diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index cae9a192..0a035e9c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,8 +54,13 @@ jobs: name: Generate build matrix runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + matrix-x86_64: ${{ steps.set-matrix.outputs.matrix-x86_64 }} + matrix-i686: ${{ steps.set-matrix.outputs.matrix-i686 }} + matrix-aarch64: ${{ steps.set-matrix.outputs.matrix-aarch64 }} crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }} + dependencies-build-matrix-x86_64: ${{ steps.set-matrix.outputs.dependencies-build-matrix-x86_64 }} + dependencies-build-matrix-i686: ${{ steps.set-matrix.outputs.dependencies-build-matrix-i686 }} + dependencies-build-matrix-aarch64: ${{ steps.set-matrix.outputs.dependencies-build-matrix-aarch64 }} any_builds: ${{ steps.set-matrix.outputs.any_builds }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -92,16 +97,23 @@ jobs: - name: Generate build matrix id: set-matrix run: | - uv run ci-matrix.py --platform windows --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json + uv run ci-matrix.py --platform windows --facet-by-triple --labels "${STEPS_GET_LABELS_OUTPUTS_LABELS}" ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json # Extract python-build matrix - echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT + echo "matrix-x86_64=$(jq -c '."python-build"?["x86_64-pc-windows-msvc"]? // {"include": []}' matrix.json)" >> $GITHUB_OUTPUT + echo "matrix-i686=$(jq -c '."python-build"?["i686-pc-windows-msvc"]? // {"include": []}' matrix.json)" >> $GITHUB_OUTPUT + echo "matrix-aarch64=$(jq -c '."python-build"?["aarch64-pc-windows-msvc"]? // {"include": []}' matrix.json)" >> $GITHUB_OUTPUT + echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT + echo "dependencies-build-matrix-x86_64=$(jq -c '."dependencies-build"?["x86_64-pc-windows-msvc"]? // {"include": []}' matrix.json)" >> $GITHUB_OUTPUT + echo "dependencies-build-matrix-i686=$(jq -c '."dependencies-build"?["i686-pc-windows-msvc"]? // {"include": []}' matrix.json)" >> $GITHUB_OUTPUT + echo "dependencies-build-matrix-aarch64=$(jq -c '."dependencies-build"?["aarch64-pc-windows-msvc"]? // {"include": []}' matrix.json)" >> $GITHUB_OUTPUT + # Display the matrix for debugging too cat matrix.json | jq - if jq -e '."python-build".include | length > 0' matrix.json > /dev/null; then + if jq -e '."python-build" | length > 0' matrix.json > /dev/null; then # Build matrix has entries echo "any_builds=true" >> $GITHUB_OUTPUT else @@ -111,21 +123,23 @@ jobs: env: STEPS_GET_LABELS_OUTPUTS_LABELS: ${{ steps.get-labels.outputs.labels }} - build: + + build-python-x86_64: timeout-minutes: 90 needs: - generate-matrix - crate-build + - build-dependencies-x86_64 # Permissions used for actions/attest-build-provenance permissions: id-token: write attestations: write runs-on: ${{ matrix.runner }} strategy: - matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-x86_64) }} fail-fast: false name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }} - steps: + steps: &build-python-steps - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 @@ -152,10 +166,17 @@ jobs: run: | uv run build.py --help + - name: Download build dependencies + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: build-deps-${{ matrix.target_triple }}-${{ matrix.build_options}} + path: build + - name: Build if: ${{ ! matrix.dry-run }} shell: cmd run: | + DIR IF "%MATRIX_VS_VERSION%"=="2026" ( call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\%MATRIX_VCVARS%" ) ELSE ( @@ -185,3 +206,111 @@ jobs: run: | $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative .\pythonbuild.exe validate-distribution --run $Dists + + build-python-i686: + timeout-minutes: 90 + needs: + - generate-matrix + - crate-build + - build-dependencies-i686 + # Permissions used for actions/attest-build-provenance + permissions: + id-token: write + attestations: write + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-i686) }} + fail-fast: false + name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }} + steps: *build-python-steps + + build-python-aarch64: + timeout-minutes: 90 + needs: + - generate-matrix + - crate-build + - build-dependencies-aarch64 + # Permissions used for actions/attest-build-provenance + permissions: + id-token: write + attestations: write + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-aarch64) }} + fail-fast: false + name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }} + steps: *build-python-steps + + build-dependencies-x86_64: + timeout-minutes: 20 + needs: + - generate-matrix + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.dependencies-build-matrix-x86_64) }} + fail-fast: false + name: build-dependencies / ${{ matrix.target_triple }} / ${{ matrix.build_options }} + steps: &build-dependencies-steps + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Cygwin Environment + uses: cygwin/cygwin-install-action@f2009323764960f80959895c7bc3bb30210afe4d # v6 + with: + packages: autoconf automake libtool + + - name: Set up uv + uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 + with: + enable-cache: false + + # We need to do this before we activate the VC++ environment or else binary packages + # don't get compiled properly. + - name: Bootstrap Python environment + run: | + uv run build.py --help + + - name: Build + if: ${{ ! matrix.dry-run }} + shell: cmd + run: | + IF "%MATRIX_VS_VERSION%"=="2026" ( + call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\%MATRIX_VCVARS%" + ) ELSE ( + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\%MATRIX_VCVARS%" + ) + uv run build.py --only-deps --sh c:\cygwin\bin\sh.exe --options %MATRIX_BUILD_OPTIONS% --vs %MATRIX_VS_VERSION% + env: + MATRIX_VCVARS: ${{ matrix.vcvars }} + MATRIX_BUILD_OPTIONS: ${{ matrix.build_options }} + MATRIX_VS_VERSION: ${{ matrix.vs_version }} + + - name: Upload Build Dependencies + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: build-deps-${{ matrix.target_triple }}-${{ matrix.build_options }} + path: build/*.tar + + build-dependencies-i686: + timeout-minutes: 20 + needs: + - generate-matrix + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.dependencies-build-matrix-i686) }} + fail-fast: false + name: build-dependencies / ${{ matrix.target_triple }} / ${{ matrix.build_options }} + steps: *build-dependencies-steps + + build-dependencies-aarch64: + timeout-minutes: 20 + needs: + - generate-matrix + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.dependencies-build-matrix-aarch64) }} + fail-fast: false + name: build-dependencies / ${{ matrix.target_triple }} / ${{ matrix.build_options }} + steps: *build-dependencies-steps diff --git a/ci-matrix.py b/ci-matrix.py index c88f09b2..cdd6e646 100644 --- a/ci-matrix.py +++ b/ci-matrix.py @@ -9,6 +9,7 @@ import argparse import json import sys +from collections import defaultdict from typing import Any import yaml @@ -127,6 +128,30 @@ def generate_docker_matrix_entries( return matrix_entries +def generate_dependencies_build_matrix_entries( + python_entries: list[dict[str, str]], + platform_filter: str | None = None, +) -> list[dict[str, str]]: + """Generate matrix entries for dependencies builds based on python build matrix.""" + include_keys = { + "arch", + "target_triple", + "platform", + "runner", + "build_options", + "vcvars", + "vs_version", + } + dep_build_pairs = set() + for entry in python_entries: + platform = entry["platform"] + if platform_filter and platform != platform_filter: + continue + pairs = tuple((k, v) for k, v in entry.items() if k in include_keys) + dep_build_pairs.add(pairs) + return [dict(pairs) for pairs in dep_build_pairs] + + def generate_crate_build_matrix_entries( python_entries: list[dict[str, str]], runners: dict[str, Any], @@ -212,6 +237,17 @@ def generate_python_build_matrix_entries( return matrix_entries +def facet_by_triple(entries: list[dict[str, str]]) -> dict[str, dict[str, list]]: + """Split entries into sections based upon the target_triple key.""" + by_triple = defaultdict(list) + for entry in entries: + by_triple[entry["target_triple"]].append(entry) + entries_by_triple = { + triple: {"include": entries} for triple, entries in by_triple.items() + } + return entries_by_triple + + def find_runner(runners: dict[str, Any], platform: str, arch: str, free: bool) -> str: # Find a matching platform first match_platform = [ @@ -333,12 +369,18 @@ def parse_args() -> argparse.Namespace: choices=["darwin", "linux", "windows"], help="Filter matrix entries by platform", ) - parser.add_argument( + group = parser.add_mutually_exclusive_group() + group.add_argument( "--max-shards", type=int, default=0, help="The maximum number of shards allowed; set to zero to disable ", ) + group.add_argument( + "--facet-by-triple", + action="store_true", + help="Facet the python-build and dependencies-build entries by target_triple", + ) parser.add_argument( "--labels", help="Comma-separated list of labels to filter by (e.g., 'platform:darwin,python:3.13,build:debug'), all must match.", @@ -355,7 +397,13 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--matrix-type", - choices=["python-build", "docker-build", "crate-build", "all"], + choices=[ + "python-build", + "docker-build", + "crate-build", + "dependencies-build", + "all", + ], default="all", help="Which matrix types to generate (default: all)", ) @@ -443,6 +491,22 @@ def main() -> None: ) result["crate-build"] = {"include": crate_entries} + # Generate dependencies-build matrix if requested + if args.matrix_type in ["dependencies-build", "all"]: + deps_entries = generate_dependencies_build_matrix_entries( + python_entries, + args.platform, + ) + result["dependencies-build"] = {"include": deps_entries} + + # facet + if args.facet_by_triple: + result["python-build"] = facet_by_triple(result["python-build"]["include"]) + if "dependencies-build" in result: + result["dependencies-build"] = facet_by_triple( + result["dependencies-build"]["include"] + ) + print(json.dumps(result)) diff --git a/cpython-windows/build.py b/cpython-windows/build.py index 3b27e679..87c04539 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -1910,6 +1910,11 @@ def main() -> None: default="10.0.26100.0", help="Windows SDK version to build with", ) + parser.add_argument( + "--only-deps", + action="store_true", + help="Build only dependencies, not CPython", + ) args = parser.parse_args() build_options = args.options @@ -1964,6 +1969,10 @@ def main() -> None: libffi_archive, ) + # Stop if only building dependencies + if args.only_deps: + return + LOG_PREFIX[0] = "cpython" tar_path = build_cpython( args.python,