diff --git a/.github/workflows/write-dockerfile.sh b/.github/workflows/write-dockerfile.sh index a52681d7c73..05cdd5db3a0 100755 --- a/.github/workflows/write-dockerfile.sh +++ b/.github/workflows/write-dockerfile.sh @@ -277,7 +277,6 @@ FROM with-system-packages AS bootstrapped RUN rm -rf /new /sage/.git $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap conftest.py configure_wrapper configure.ac sage .homebrew-build-env tox.ini .gitignore pyproject.toml meson.build meson.options /new/ $ADD config/config.rpath /new/config/config.rpath -$ADD src/doc/bootstrap /new/src/doc/bootstrap $ADD src/meson.build /new/src/ $ADD src/bin /new/src/bin $ADD m4 /new/m4 @@ -288,8 +287,8 @@ $ADD tools /new/tools $ADD .github/workflows /.github/workflows RUN if [ -d /sage ]; then \\ echo "### Incremental build from \$(cat /sage/VERSION.txt)" && \\ - printf '/src/*\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/VERSION.txt\n' >> /sage/.gitignore && \\ - printf '/src/*\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/VERSION.txt\n' >> /new/.gitignore && \\ + printf '/src/*\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/VERSION.txt\n' >> /sage/.gitignore && \\ + printf '/src/*\n\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/VERSION.txt\n' >> /new/.gitignore && \\ if ! (cd /new && /.github/workflows/retrofit-worktree.sh worktree-image /sage); then \\ echo "retrofit-worktree.sh failed, falling back to replacing /sage"; \\ for a in local logs; do \\ diff --git a/Makefile b/Makefile index d9bc6ae1070..a5e80e801f4 100644 --- a/Makefile +++ b/Makefile @@ -364,14 +364,14 @@ CONFIGURE_DEPENDENCIES = \ build/pkgs/*/spkg-install build/pkgs/*/spkg-install.in # SPKG_INFO_DEPENDENCIES is the list of files that influence the run of 'sage-spkg-info' and hence -# the generation of the files generated in 'src/doc' by 'src/doc/bootstrap'. +# the generation of the files generated 'tools/bootstrap-docs.py'. SPKG_INFO_DEPENDENCIES = \ build/pkgs/*/type build/pkgs/*/SPKG.rst \ build/pkgs/*/requirements.txt \ build/pkgs/*/version_requirements.txt build/pkgs/*/package-version.txt \ build/pkgs/*/distros/*.txt -configure: bootstrap src/doc/bootstrap $(CONFIGURE_DEPENDENCIES) $(SPKG_INFO_DEPENDENCIES) +configure: bootstrap tools/bootstrap-docs.py $(CONFIGURE_DEPENDENCIES) $(SPKG_INFO_DEPENDENCIES) ./bootstrap -d install: all diff --git a/build/pkgs/sage_docbuild/dependencies b/build/pkgs/sage_docbuild/dependencies index c21e4b287e3..4797a8d5946 100644 --- a/build/pkgs/sage_docbuild/dependencies +++ b/build/pkgs/sage_docbuild/dependencies @@ -1 +1 @@ - sphinx $(SAGE_ROOT)/pkgs/sage-docbuild/sage_docbuild/*.py $(SAGE_ROOT)/pkgs/sage-docbuild/sage_docbuild/ext/*.py | $(PYTHON_TOOLCHAIN) sagelib $(PYTHON) + sphinx | $(PYTHON_TOOLCHAIN) sagelib $(PYTHON) diff --git a/build/pkgs/sage_setup/dependencies b/build/pkgs/sage_setup/dependencies index 497a8118c28..f6cf2bd29d7 100644 --- a/build/pkgs/sage_setup/dependencies +++ b/build/pkgs/sage_setup/dependencies @@ -1,4 +1,4 @@ - cython pkgconfig jinja2 $(SAGE_ROOT)/pkgs/sage-setup/sage_setup/*.py $(SAGE_ROOT)/pkgs/sage-setup/sage_setup/autogen/interpreters/internal/specs/*.py $(SAGE_ROOT)/pkgs/sage-setup/sage_setup/command/*.py | $(PYTHON_TOOLCHAIN) $(PYTHON) + cython pkgconfig jinja2 | $(PYTHON_TOOLCHAIN) $(PYTHON) ---------- All lines of this file are ignored except the first. diff --git a/build/sage_bootstrap/app.py b/build/sage_bootstrap/app.py index 53bbde023b8..e4e48a8d2ee 100644 --- a/build/sage_bootstrap/app.py +++ b/build/sage_bootstrap/app.py @@ -23,22 +23,22 @@ # **************************************************************************** +import logging import os import re -import logging + log = logging.getLogger() from collections import defaultdict -from sage_bootstrap.package import Package -from sage_bootstrap.tarball import Tarball, FileNotMirroredError -from sage_bootstrap.updater import ChecksumUpdater, PackageUpdater from sage_bootstrap.creator import PackageCreator -from sage_bootstrap.pypi import PyPiVersion, PyPiNotFound, PyPiError -from sage_bootstrap.fileserver import FileServer -from sage_bootstrap.expand_class import PackageClass from sage_bootstrap.env import SAGE_DISTFILES - +from sage_bootstrap.expand_class import PackageClass +from sage_bootstrap.fileserver import FileServer +from sage_bootstrap.package import Package +from sage_bootstrap.pypi import PyPiError, PyPiNotFound, PyPiVersion +from sage_bootstrap.tarball import FileNotMirroredError, Tarball +from sage_bootstrap.updater import ChecksumUpdater, PackageUpdater # Approximation of https://peps.python.org/pep-0508/#names dependency specification dep_re = re.compile('^ *([-A-Z0-9._]+)', re.IGNORECASE) @@ -181,7 +181,7 @@ def dependencies(self, *package_classes, **kwds): # Dependencies like $(BLAS) print(indent2 + "- {0}".format(dep)) elif format == 'rst' and Package(dep).has_file('SPKG.rst'): - # This RST label is set in src/doc/bootstrap + # This RST label is set in tools/bootstrap-docs.py print(indent2 + "- :ref:`spkg_{0}`".format(dep)) else: print(indent2 + "- {0}".format(dep)) @@ -220,7 +220,7 @@ def apropos(self, incorrect_name): Did you mean: cython, ipython, python2, python3, patch? """ log.debug('Apropos for %s', incorrect_name) - from sage_bootstrap.levenshtein import Levenshtein, DistanceExceeded + from sage_bootstrap.levenshtein import DistanceExceeded, Levenshtein levenshtein = Levenshtein(5) names = [] for pkg in Package.all(): diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 2d40d4915df..3f662882eb0 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -18,6 +18,7 @@ import logging import os import re +from pathlib import Path from sage_bootstrap.env import SAGE_ROOT @@ -25,22 +26,21 @@ class Package(object): - def __new__(cls, package_name): if package_name.startswith("pypi/") or package_name.startswith("generic/"): package_name = "pkg:" + package_name if package_name.startswith("pkg:"): - package_name = package_name.replace('_', '-').lower() + package_name = package_name.replace("_", "-").lower() if package_name.startswith("pkg:generic/"): # fast path try: - pkg = cls(package_name[len("pkg:generic/"):].replace('-', '_')) + pkg = cls(package_name[len("pkg:generic/") :].replace("-", "_")) if pkg.purl == package_name: return pkg # assume unique except Exception: pass elif package_name.startswith("pkg:pypi/"): # fast path try: - pkg = cls(package_name[len("pkg:pypi/"):].replace('-', '_')) + pkg = cls(package_name[len("pkg:pypi/") :].replace("-", "_")) if pkg.purl == package_name: return pkg # assume unique except Exception: @@ -48,7 +48,7 @@ def __new__(cls, package_name): for pkg in cls.all(): if pkg.purl == package_name: return pkg # assume unique - raise ValueError('no package for PURL {0}'.format(package_name)) + raise ValueError("no package for PURL {0}".format(package_name)) self = object.__new__(cls) self.__init__(package_name) return self @@ -69,14 +69,21 @@ def __init__(self, package_name): -- ``package_name`` -- string. Name of the package. The Sage convention is that all package names are lower case. """ - if any(package_name.startswith(prefix) - for prefix in ["pkg:", "pypi/", "generic"]): + if any( + package_name.startswith(prefix) for prefix in ["pkg:", "pypi/", "generic"] + ): # Already initialized return if package_name != package_name.lower(): - raise ValueError('package names should be lowercase, got {0}'.format(package_name)) - if '-' in package_name: - raise ValueError('package names use underscores, not dashes, got {0}'.format(package_name)) + raise ValueError( + "package names should be lowercase, got {0}".format(package_name) + ) + if "-" in package_name: + raise ValueError( + "package names use underscores, not dashes, got {0}".format( + package_name + ) + ) self.__name = package_name self.__tarball = None @@ -89,7 +96,7 @@ def __init__(self, package_name): self._init_trees() def __repr__(self): - return 'Package {0}'.format(self.name) + return "Package {0}".format(self.name) @property def name(self): @@ -142,6 +149,7 @@ def tarball(self): """ if self.__tarball is None: from sage_bootstrap.tarball import Tarball + self.__tarball = Tarball(self.tarball_filename, package=self) return self.__tarball @@ -157,9 +165,9 @@ def _substitute_variables_once(self, pattern): - the string with the substitution done or the original string - whether a substitution was done """ - for var in ('VERSION_MAJOR', 'VERSION_MINOR', 'VERSION_MICRO', 'VERSION'): + for var in ("VERSION_MAJOR", "VERSION_MINOR", "VERSION_MICRO", "VERSION"): # As VERSION is a substring of the other three, it needs to be tested last. - dollar_brace_var = '${' + var + '}' + dollar_brace_var = "${" + var + "}" if dollar_brace_var in pattern: value = getattr(self, var.lower()) return pattern.replace(dollar_brace_var, value, 1), True @@ -281,7 +289,7 @@ def version_major(self): String. The package's major version. """ - return self.version.split('.')[0] + return self.version.split(".")[0] @property def version_minor(self): @@ -292,7 +300,7 @@ def version_minor(self): String. The package's minor version. """ - return self.version.split('.')[1] + return self.version.split(".")[1] @property def version_micro(self): @@ -303,7 +311,7 @@ def version_micro(self): String. The package's micro version. """ - return self.version.split('.')[2] + return self.version.split(".")[2] @property def patchlevel(self): @@ -347,14 +355,14 @@ def source(self): Return the package source type """ if self.__requirements is not None: - return 'pip' + return "pip" if self.tarball_filename: - if self.tarball_filename.endswith('.whl'): - return 'wheel' - return 'normal' - if self.has_file('spkg-install') or self.has_file('spkg-install.in'): - return 'script' - return 'none' + if self.tarball_filename.endswith(".whl"): + return "wheel" + return "normal" + if self.has_file("spkg-install") or self.has_file("spkg-install.in"): + return "script" + return "none" @property def trees(self): @@ -368,10 +376,10 @@ def trees(self): if self.__trees is not None: return self.__trees if self.__version_requirements is not None: - return 'SAGE_VENV' + return "SAGE_VENV" if self.__requirements is not None: - return 'SAGE_VENV' - return 'SAGE_LOCAL' + return "SAGE_VENV" + return "SAGE_LOCAL" @property def purl(self): @@ -387,8 +395,8 @@ def purl(self): """ dist = self.distribution_name if dist: - return 'pkg:pypi/' + dist.lower().replace('_', '-') - return 'pkg:generic/' + self.name.replace('_', '-') + return "pkg:pypi/" + dist.lower().replace("_", "-") + return "pkg:generic/" + self.name.replace("_", "-") @property def distribution_name(self): @@ -396,17 +404,17 @@ def distribution_name(self): Return the Python distribution name or ``None`` for non-Python packages """ if self.__requirements is not None: - for line in self.__requirements.split('\n'): + for line in self.__requirements.split("\n"): line = line.strip() - if line.startswith('#'): + if line.startswith("#"): continue for part in line.split(): return part if self.__version_requirements is None: return None - for line in self.__version_requirements.split('\n'): + for line in self.__version_requirements.split("\n"): line = line.strip() - if line.startswith('#'): + if line.startswith("#"): continue for part in line.split(): return part @@ -418,14 +426,17 @@ def dependencies(self): Return a list of strings, the package names of the (ordinary) dependencies """ # after a '|', we have order-only dependencies - return self.__dependencies.partition('|')[0].strip().split() + return self.__dependencies.partition("|")[0].strip().split() @property def dependencies_order_only(self): """ Return a list of strings, the package names of the order-only dependencies """ - return self.__dependencies.partition('|')[2].strip().split() + self.__dependencies_order_only.strip().split() + return ( + self.__dependencies.partition("|")[2].strip().split() + + self.__dependencies_order_only.strip().split() + ) @property def dependencies_optional(self): @@ -440,7 +451,7 @@ def dependencies_runtime(self): Return a list of strings, the package names of the runtime dependencies """ # after a '|', we have order-only build dependencies - return self.__dependencies.partition('|')[0].strip().split() + return self.__dependencies.partition("|")[0].strip().split() @property def dependencies_check(self): @@ -457,16 +468,16 @@ def all(cls): """ Return all packages """ - base = os.path.join(SAGE_ROOT, 'build', 'pkgs') + base = os.path.join(SAGE_ROOT, "build", "pkgs") for subdir in os.listdir(base): path = os.path.join(base, subdir) if not os.path.isfile(os.path.join(path, "type")): - log.debug('%s has no type', subdir) + log.debug("%s has no type", subdir) continue try: yield cls(subdir) except Exception: - log.error('Failed to open %s', subdir) + log.error("Failed to open %s", subdir) raise @property @@ -474,7 +485,7 @@ def path(self): """ Return the package directory """ - return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name) + return os.path.join(SAGE_ROOT, "build", "pkgs", self.name) def has_file(self, filename): """ @@ -496,8 +507,10 @@ def line_count_file(self, filename): if os.path.islink(path): return 1 if os.path.isdir(path): - return sum(self.line_count_file(os.path.join(filename, entry)) - for entry in os.listdir(path)) + return sum( + self.line_count_file(os.path.join(filename, entry)) + for entry in os.listdir(path) + ) try: with open(path, "rb") as f: return len(list(f)) @@ -508,11 +521,11 @@ def _init_checksum(self): """ Load the checksums from the appropriate ``checksums.ini`` file """ - checksums_ini = os.path.join(self.path, 'checksums.ini') - assignment = re.compile('(?P[a-zA-Z0-9_]*)=(?P.*)') + checksums_ini = os.path.join(self.path, "checksums.ini") + assignment = re.compile("(?P[a-zA-Z0-9_]*)=(?P.*)") result = dict() try: - with open(checksums_ini, 'rt') as f: + with open(checksums_ini, "rt") as f: for line in f.readlines(): match = assignment.match(line) if match is None: @@ -521,18 +534,18 @@ def _init_checksum(self): result[var] = value except IOError: pass - self.__sha1 = result.get('sha1', None) - self.__sha256 = result.get('sha256', None) - self.__tarball_pattern = result.get('tarball', None) - self.__tarball_upstream_url_pattern = result.get('upstream_url', None) + self.__sha1 = result.get("sha1", None) + self.__sha256 = result.get("sha256", None) + self.__tarball_pattern = result.get("tarball", None) + self.__tarball_upstream_url_pattern = result.get("upstream_url", None) # Name of the directory containing the checksums.ini file self.__tarball_package_name = os.path.realpath(checksums_ini).split(os.sep)[-2] - VERSION_PATCHLEVEL = re.compile(r'(?P.*)\.p(?P[0-9]+)') + VERSION_PATCHLEVEL = re.compile(r"(?P.*)\.p(?P[0-9]+)") def _init_version(self): try: - with open(os.path.join(self.path, 'package-version.txt')) as f: + with open(os.path.join(self.path, "package-version.txt")) as f: package_version = f.read().strip() except IOError: self.__version = None @@ -543,56 +556,83 @@ def _init_version(self): self.__version = package_version self.__patchlevel = -1 else: - self.__version = match.group('version') - self.__patchlevel = int(match.group('patchlevel')) + self.__version = match.group("version") + self.__patchlevel = int(match.group("patchlevel")) def _init_type(self): - with open(os.path.join(self.path, 'type')) as f: + with open(os.path.join(self.path, "type")) as f: package_type = f.read().strip() - assert package_type in [ - 'base', 'standard', 'optional', 'experimental' - ] + assert package_type in ["base", "standard", "optional", "experimental"] self.__type = package_type def _init_version_requirements(self): try: - with open(os.path.join(self.path, 'version_requirements.txt')) as f: + with open(os.path.join(self.path, "version_requirements.txt")) as f: self.__version_requirements = f.read().strip() except IOError: self.__version_requirements = None def _init_requirements(self): try: - with open(os.path.join(self.path, 'requirements.txt')) as f: + with open(os.path.join(self.path, "requirements.txt")) as f: self.__requirements = f.read().strip() except IOError: self.__requirements = None def _init_dependencies(self): try: - with open(os.path.join(self.path, 'dependencies')) as f: - self.__dependencies = f.readline().partition('#')[0].strip() + with open(os.path.join(self.path, "dependencies")) as f: + self.__dependencies = f.readline().partition("#")[0].strip() except IOError: - self.__dependencies = '' + self.__dependencies = "" try: - with open(os.path.join(self.path, 'dependencies_check')) as f: - self.__dependencies_check = f.readline().partition('#')[0].strip() + with open(os.path.join(self.path, "dependencies_check")) as f: + self.__dependencies_check = f.readline().partition("#")[0].strip() except IOError: - self.__dependencies_check = '' + self.__dependencies_check = "" try: - with open(os.path.join(self.path, 'dependencies_optional')) as f: - self.__dependencies_optional = f.readline().partition('#')[0].strip() + with open(os.path.join(self.path, "dependencies_optional")) as f: + self.__dependencies_optional = f.readline().partition("#")[0].strip() except IOError: - self.__dependencies_optional = '' + self.__dependencies_optional = "" try: - with open(os.path.join(self.path, 'dependencies_order_only')) as f: - self.__dependencies_order_only = f.readline().partition('#')[0].strip() + with open(os.path.join(self.path, "dependencies_order_only")) as f: + self.__dependencies_order_only = f.readline().partition("#")[0].strip() except IOError: - self.__dependencies_order_only = '' + self.__dependencies_order_only = "" def _init_trees(self): try: - with open(os.path.join(self.path, 'trees.txt')) as f: - self.__trees = f.readline().partition('#')[0].strip() + with open(os.path.join(self.path, "trees.txt")) as f: + self.__trees = f.readline().partition("#")[0].strip() except IOError: self.__trees = None + + def read_system_packages(self, system: str) -> list[str]: + """Return the distro package names for ``pkg`` on ``system``.""" + + rel_path = f"distros/{system}.txt" + if not self.has_file(rel_path): + return [] + python_minor = os.environ.get("PYTHON_MINOR", "") + packages: list[str] = [] + path = Path(self.path) / rel_path + for raw_line in path.read_text().splitlines(): + line = raw_line.split("#", 1)[0] + if "${PYTHON_MINOR}" in line: + line = line.replace("${PYTHON_MINOR}", python_minor) + stripped = line.strip() + if not stripped: + continue + packages.extend(stripped.split()) + return packages + + def is_python_package(self) -> bool: + """Return whether this is a Python package.""" + if self.name == "meson": + # 'meson-python' is the actual Python package + return False + + return ( + self.__requirements is not None or self.__version_requirements is not None + ) diff --git a/src/doc/bootstrap b/src/doc/bootstrap deleted file mode 100755 index 5d57254ac55..00000000000 --- a/src/doc/bootstrap +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash - -######################################################################## -# Regenerate auto-generated files, using information in SAGE_ROOT/build/ -# -# This script is run by SAGE_ROOT/bootstrap as part of the bootstrapping phase -# (before configure, before creating source distributions). -# -# The BOOTSTRAP_QUIET variable is set by the top-level -# bootstrap script and controls how verbose we are. -######################################################################## - -set -e - -if [ -z "$SAGE_ROOT" ]; then - echo Please run the top-level bootstrap script of the Sage distribution. - exit 1 -fi - -cd "$SAGE_ROOT" -export PATH=build/bin:$PATH - -# Get target directory from first command-line argument -TARGET_DIR="${1:-$SAGE_ROOT/src/doc}" - -OUTPUT_DIR="$TARGET_DIR/en/installation" -mkdir -p "$OUTPUT_DIR" - -shopt -s extglob - -RECOMMENDED_SPKG_PATTERN="@(_recommended$(for a in $(head -n 1 build/pkgs/_recommended/dependencies); do echo -n "|"$a; done))" -DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))" - -for SYSTEM in arch debian fedora homebrew opensuse void; do - SYSTEM_PACKAGES= - OPTIONAL_SYSTEM_PACKAGES= - SAGELIB_SYSTEM_PACKAGES= - SAGELIB_OPTIONAL_SYSTEM_PACKAGES= - RECOMMENDED_SYSTEM_PACKAGES= - DEVELOP_SYSTEM_PACKAGES= - for PKG_BASE in $(sage-package list --has-file distros/$SYSTEM.txt); do - PKG_SCRIPTS=build/pkgs/$PKG_BASE - PKG_TYPE=$(cat $PKG_SCRIPTS/type) - PKG_SYSTEM_PACKAGES=$(sage-get-system-packages $SYSTEM $PKG_BASE) - if [ -n "PKG_SYSTEM_PACKAGES" ]; then - if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then - case "$PKG_BASE:$PKG_TYPE" in - *:standard) - SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - $RECOMMENDED_SPKG_PATTERN:*) - RECOMMENDED_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - *) - OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - esac - else - case "$PKG_BASE:$PKG_TYPE" in - $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - *:standard) - SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - *) - SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" - ;; - esac - fi - fi - done - - if [ "${BOOTSTRAP_QUIET}" = "no" ]; then - echo >&2 $0:$LINENO: installing "$OUTPUT_DIR"/$SYSTEM"*.txt" - fi - echo "$(sage-print-system-package-command $SYSTEM --prompt --wrap --sudo install $(echo $(echo $SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM.txt - echo "$(sage-print-system-package-command $SYSTEM --prompt --wrap --sudo install $(echo $(echo $OPTIONAL_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-optional.txt - echo "$(sage-print-system-package-command $SYSTEM --prompt --wrap --sudo install $(echo $(echo $RECOMMENDED_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-recommended.txt - echo "$(sage-print-system-package-command $SYSTEM --prompt --wrap --sudo install $(echo $(echo $DEVELOP_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-develop.txt -done - -OUTPUT_DIR="$TARGET_DIR/en/reference/spkg" -mkdir -p "$OUTPUT_DIR" -if [ "${BOOTSTRAP_QUIET}" = "no" ]; then - echo >&2 $0:$LINENO: installing "$OUTPUT_DIR"/"*.rst" -fi -(cat < "$OUTPUT_DIR"/index_standard.rst -(cat < "$OUTPUT_DIR"/index_optional.rst -for PKG_BASE in $(sage-package list --has-file SPKG.rst | grep '^sagemath_'); do - echo "* :ref:\`spkg_$PKG_BASE\`" -done > "$OUTPUT_DIR"/index_sagemath.rst -(cat < "$OUTPUT_DIR"/index_experimental.rst -(cat < "$OUTPUT_DIR"/index_alph.rst -sage-package list --has-file SPKG.rst | OUTPUT_DIR=$OUTPUT_DIR OUTPUT_RST=1 xargs -P $(nproc 2> /dev/null || echo 8) -n 1 sage-spkg-info diff --git a/src/doc/meson.build b/src/doc/meson.build index d20e287caa9..c973d8afd8a 100644 --- a/src/doc/meson.build +++ b/src/doc/meson.build @@ -30,7 +30,11 @@ subdir('ja') doc_bootstrap = custom_target( 'bootstrap', output: ['autogen'], - command: [files('bootstrap'), meson.current_build_dir()], + command: [ + py, + files('../../tools/bootstrap-docs.py'), + meson.current_build_dir(), + ], env: {'SAGE_ROOT': root}, # doc_src is not really a dependency of the bootstrap, but we want to make sure # that all the source files are present before running the actual doc build diff --git a/tools/bootstrap-docs.py b/tools/bootstrap-docs.py new file mode 100644 index 00000000000..e86cc985566 --- /dev/null +++ b/tools/bootstrap-docs.py @@ -0,0 +1,574 @@ +#!/usr/bin/env python3 + +"""Generate some of the documentation sources.""" + +from __future__ import annotations + +import inspect +import re +import shlex +import sys +import textwrap +from pathlib import Path +from typing import Sequence + +# So that Python can find the sage_bootstrap package +sys.path.insert(0, str(Path(__file__).parent.parent / "build")) + +from sage_bootstrap.expand_class import PackageClass +from sage_bootstrap.package import Package + + +def log_install(target_pattern: str) -> None: + frame = inspect.currentframe() + caller = frame.f_back if frame else None + lineno = caller.f_lineno if caller else 0 + script = Path(__file__).name + print(f"{script}:{lineno}: installing {target_pattern}") + + +def write_text(path: Path, content: str) -> None: + """Write ``content`` to ``path``, always terminating with a newline.""" + + path.parent.mkdir(parents=True, exist_ok=True) + if not content.endswith("\n"): + content += "\n" + path.write_text(content, encoding="utf-8") + + +def has_python_package_check(pkg) -> bool: + """Detect whether ``pkg`` relies on ``SAGE_PYTHON_PACKAGE_CHECK``.""" + + if not pkg.has_file("spkg-configure.m4"): + return False + spkg_configure = Path(pkg.path) / "spkg-configure.m4" + return "SAGE_PYTHON_PACKAGE_CHECK" in spkg_configure.read_text() + + +SYSTEM_COMMANDS: dict[str, Sequence[str]] = { + "arch": ("sudo", "pacman", "-S"), + "debian": ("sudo", "apt-get", "install"), + "fedora": ("sudo", "dnf", "install"), + "homebrew": ("brew", "install"), + "opensuse": ("sudo", "zypper", "install"), + "void": ("sudo", "xbps-install"), + "alpine": ("apk", "add"), + "conda": ("conda", "install"), + "freebsd": ("sudo", "pkg", "install"), + "gentoo": ("sudo", "emerge"), + "macports": ("sudo", "port", "install"), + "nix": ("nix-env", "--install"), + "openbsd": ("sudo", "pkg_add"), + "slackware": ("sudo", "slackpkg", "install"), +} + + +def format_shell_command(tokens: Sequence[str], wrap: int | None = 78) -> str: + """Format a shell command.""" + + if not tokens: + return "" + quoted = " ".join([shlex.quote(token) for token in tokens]) + if wrap is None: + return "$ " + quoted + " " + lines = textwrap.wrap( + quoted, + width=wrap, + initial_indent=" $ ", + subsequent_indent=" ", + break_long_words=False, + break_on_hyphens=False, + ) + lines = [line + " " for line in lines] + return " \\\n".join(lines) + + +def collect_installation_packages( + system: str, + recommended: set[str], + develop: set[str], +) -> dict[str, list[str]]: + """Collect distro packages grouped by documentation category.""" + + categories = { + "standard": set(), + "optional": set(), + "recommended": set(), + "develop": set(), + } + + selector = PackageClass(":all:", has_files=[f"distros/{system}.txt"]) + for pkg_name in selector.names: + pkg = Package(pkg_name) + system_packages = pkg.read_system_packages(system) + if not system_packages: + continue + if pkg_name == "_develop": + categories["develop"].update(system_packages) + continue + if pkg_name == "_recommended": + categories["recommended"].update(system_packages) + continue + if pkg.is_python_package(): + continue + has_configure = pkg.has_file("spkg-configure.m4") + if has_configure: + if pkg.type == "standard": + categories["standard"].update(system_packages) + elif pkg_name in develop: + categories["develop"].update(system_packages) + elif pkg_name in recommended: + categories["recommended"].update(system_packages) + else: + categories["optional"].update(system_packages) + elif pkg_name in develop: + categories["develop"].update(system_packages) + + return { + key: sorted(pkg for pkg in value if pkg) for key, value in categories.items() + } + + +def generate_installation_docs(base_dir: Path) -> None: + """Generate ``installation`` command snippets.""" + + base_dir.mkdir(parents=True, exist_ok=True) + recommended = Package("_recommended").dependencies + develop = Package("_develop").dependencies + + for system, install_command in SYSTEM_COMMANDS.items(): + log_install(f"{base_dir}/{system}*.txt") + categories = collect_installation_packages( + system, + recommended, + develop, + ) + for suffix, key in ( + ("", "standard"), + ("-optional", "optional"), + ("-recommended", "recommended"), + ("-develop", "develop"), + ): + path = base_dir / f"{system}{suffix}.txt" + packages = categories.get(key, []) + if packages: + tokens = list(install_command) + packages + write_text(path, format_shell_command(tokens, wrap=None)) + else: + write_text(path, "") + + +def packages_for_index( + selector: str, + *, + has_files: list[str] = [], + no_files: list[str] = [], + exclude_prefix: str | None = None, +) -> list[str]: + """Collect package names suited for the index sections.""" + + names = PackageClass(selector, has_files=has_files, no_files=no_files).names + if exclude_prefix is not None: + names = [name for name in names if not name.startswith(exclude_prefix)] + return names + + +def write_index_sections( + path: Path, sections: Sequence[tuple[str, Sequence[str]]] +) -> None: + """Write an index file composed of titled sections.""" + + lines: list[str] = [] + for title, packages in sections: + lines.append(title) + lines.append("~" * len(title)) + lines.append("") + for name in packages: + lines.append(f"* :ref:`spkg_{name}`") + lines.append("") + write_text(path, "\n".join(lines)) + + +def write_bullet_list(path: Path, package_names: Sequence[str]) -> None: + """Write a simple bullet list of package references.""" + + lines = [f"* :ref:`spkg_{name}`" for name in package_names] + write_text(path, "\n".join(lines)) + + +def write_alphabetical_index(path: Path, package_names: Sequence[str]) -> None: + """Write the alphabetical ``index_alph.rst`` file.""" + + lines = [ + "", + "Details of external packages", + "============================", + "", + "Packages are in alphabetical order.", + "", + ".. default-role:: code", + "", + ".. toctree::", + " :maxdepth: 1", + "", + ] + lines.extend(f" {name}" for name in package_names) + lines.extend(["", ".. default-role::", ""]) + write_text(path, "\n".join(lines)) + + +def generate_spkg_indexes(base_dir: Path) -> None: + """Generate the ``reference/spkg`` index files.""" + + base_dir.mkdir(parents=True, exist_ok=True) + log_install(f"{base_dir}/*.rst") + + write_index_sections( + base_dir / "index_standard.rst", + ( + ( + "Mathematics", + packages_for_index( + ":standard:", + has_files=["SPKG.rst", "math"], + exclude_prefix="sagemath_", + ), + ), + ( + "Front-end, graphics, document preparation", + packages_for_index( + ":standard:", + has_files=["SPKG.rst", "front-end"], + no_files=["math"], + exclude_prefix="sagemath_", + ), + ), + ( + "Other dependencies", + packages_for_index( + ":standard:", + has_files=["SPKG.rst"], + no_files=["math", "front-end"], + exclude_prefix="sagemath_", + ), + ), + ), + ) + + write_index_sections( + base_dir / "index_optional.rst", + ( + ( + "Mathematics", + packages_for_index( + ":optional:", + has_files=["SPKG.rst", "math"], + exclude_prefix="sagemath_", + ), + ), + ( + "Front-end, graphics, document preparation", + packages_for_index( + ":optional:", + has_files=["SPKG.rst", "front-end"], + no_files=["math"], + exclude_prefix="sagemath_", + ), + ), + ( + "Other dependencies", + packages_for_index( + ":optional:", + has_files=["SPKG.rst"], + no_files=["math", "front-end"], + exclude_prefix="sagemath_", + ), + ), + ), + ) + + sagemath_packages = [ + name + for name in PackageClass(":all:", has_files=["SPKG.rst"]).names + if name.startswith("sagemath_") + ] + write_bullet_list(base_dir / "index_sagemath.rst", sagemath_packages) + + write_index_sections( + base_dir / "index_experimental.rst", + ( + ( + "Mathematics", + packages_for_index( + ":experimental:", + has_files=["SPKG.rst", "math"], + exclude_prefix="sagemath_", + ), + ), + ( + "Other dependencies", + packages_for_index( + ":experimental:", + has_files=["SPKG.rst"], + no_files=["math"], + exclude_prefix="sagemath_", + ), + ), + ), + ) + + write_alphabetical_index( + base_dir / "index_alph.rst", + PackageClass(":all:", has_files=["SPKG.rst"]).names, + ) + + +ISSUE_RE = re.compile(r"https://github.com/sagemath/sage/issues/([0-9]+)") +ARXIV_RE = re.compile(r"https://arxiv.org/abs/cs/([0-9]+)") + + +def transform_spkg_rst(package_name: str, content: str) -> str: + """Apply post-processing.""" + + lines = content.splitlines() + for idx in range(min(3, len(lines))): + lines[idx] = re.sub( + r"^ *Sage: Open Source Mathematics Software:", + f"{package_name}:", + lines[idx], + ) + text = "\n".join(lines) + text = ISSUE_RE.sub(r":issue:`\\1`", text) + text = ARXIV_RE.sub(r":arxiv:`cs/\\1`", text) + return text.rstrip() + "\n" + + +def _read_distro_packages(path: Path) -> list[str]: + """Read a distro .txt file, stripping comments and blank entries.""" + if not path.is_file(): + return [] + tokens: list[str] = [] + for line in path.read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if line: + tokens.extend(line.split()) + return tokens + + +def build_additional_sections(pkg: Package) -> str: + """Generate additional sections formerly produced by the shell pipeline. + + Sections: + Type (contents of the 'type' file if present) + Dependencies (package references) + Version Information (best-effort from a 'version' file) + Equivalent System Packages (commands to install system packages) + Configuration notes (use / non-use of system packages) + """ + + lines: list[str] = [] + + # Type section + if pkg.type: + lines.extend( + [ + "", + "Type", + "----", + "", + pkg.type, + "", + ] + ) + + # Dependencies section + lines.extend( + [ + "", + "Dependencies", + "------------", + "", + ] + ) + dependencies = pkg.dependencies + pkg.dependencies_order_only + if dependencies: + for dep in sorted(dependencies): + if dep: + if dep == 'FORCE': + # Suppress FORCE + continue + elif dep.startswith("$") or dep.startswith("sage"): + lines.append(f"- {dep}") + else: + lines.append(f"- :ref:`spkg_{dep}`") + lines.append("") + else: + lines.append("") + + # Version Information section (heuristic) + lines.extend(["Version Information", "-------------------", ""]) + for candidate in ( + "package-version.txt", + "requirements.txt", + "pyproject.toml", + "version_requirements.txt", + ): + path = Path(pkg.path) / candidate + if path.is_file(): + version_text = path.read_text().strip() + if version_text: + lines.extend( + [ + f"{candidate}::", + "", + *( + " " + line + for line in version_text.splitlines() + if not line.startswith("#") + ), + "", + ] + ) + lines.append("") + + # Equivalent System Packages + lines.extend( + [ + "Equivalent System Packages", + "--------------------------", + "", + ] + ) + distros_dir = Path(pkg.path) / "distros" + system_files: list[Path] = [] + if distros_dir.is_dir(): + system_files = sorted(p for p in distros_dir.glob("*.txt")) + have_any = False + have_repology = any(p.stem == "repology" for p in system_files) + for p in system_files: + system = p.stem + if system == "repology": + continue # defer + packages = _read_distro_packages(p) + # Heading for system (simulate tab title) + pretty = { + "alpine": "Alpine", + "arch": "Arch Linux", + "conda": "conda-forge", + "debian": "Debian/Ubuntu", + "fedora": "Fedora/Redhat/CentOS", + "freebsd": "FreeBSD", + "gentoo": "Gentoo Linux", + "homebrew": "Homebrew", + "macports": "MacPorts", + "nix": "Nixpkgs", + "openbsd": "OpenBSD", + "opensuse": "openSUSE", + "slackware": "Slackware", + "void": "Void Linux", + }.get(system, system) + if packages: + if system == "pyodide": + lines.extend( + [ + f".. tab:: {pretty}", + "", + f" install the following packages: {', '.join(packages)}", + "", + ] + ) + else: + lines.extend([f".. tab:: {pretty}", "", " .. CODE-BLOCK:: bash"]) + have_any = True + cmd_tokens = list(SYSTEM_COMMANDS.get(system, ())) + packages + if cmd_tokens: + lines.append("") + lines.append(format_shell_command(cmd_tokens)) + lines.append("") + lines.append("") + else: + lines.append("(packages: " + " ".join(packages) + ")") + lines.append("") + else: + lines.append(f".. tab:: {pretty}") + lines.append("") + lines.append(" No package needed.") + lines.append("") + + # Repology shown after others + if have_repology: + repology_file = distros_dir / "repology.txt" + packages = _read_distro_packages(repology_file) + if packages: + lines.append("") + urls = ", ".join( + f"https://repology.org/project/{p}/versions" for p in packages + ) + lines.append(f"See {urls}") + lines.append("") + have_any = True + if not have_any: + lines.append("(none known)") + lines.append("") + + # Configuration notes + if pkg.has_file("spkg-configure.m4"): + if has_python_package_check(pkg): + lines.extend( + [ + "If the system package is installed and if the (experimental) option", + "``--enable-system-site-packages`` is passed to ``./configure``, then ``./configure``", + "will check if the system package can be used.", + "", + ] + ) + else: + lines.extend( + [ + "If the system package is installed, ``./configure`` will check if it can be used.", + "", + ] + ) + elif not pkg.name.startswith("_"): + lines.extend( + [ + "However, these system packages will not be used for building Sage", + "because ``spkg-configure.m4`` has not been written for this package;", + "see :issue:`27330` for more information.", + "", + ] + ) + + return "\n".join(lines).rstrip() + "\n\n" + + +def generate_spkg_details(base_dir: Path) -> None: + """Generate per-package ``spkg`` documentation.""" + + base_dir.mkdir(parents=True, exist_ok=True) + log_install(f"{base_dir}/*.rst") + names = PackageClass(":all:", has_files=["SPKG.rst"]).names + + for name in names: + pkg = Package(name) + source = Path(pkg.path) / "SPKG.rst" + if not source.is_file(): + continue + transformed = transform_spkg_rst(name, source.read_text()) + extra = build_additional_sections(pkg) + content = f".. _spkg_{name}:\n\n{transformed}{extra}" + write_text(base_dir / f"{name}.rst", content) + + +def main() -> int: + target_dir = Path(sys.argv[1]) + + generate_installation_docs(target_dir / "en" / "installation") + generate_spkg_indexes(target_dir / "en" / "reference" / "spkg") + generate_spkg_details(target_dir / "en" / "reference" / "spkg") + + return 0 + + +if __name__ == "__main__": + sys.exit(main())