From 68b0b0d67804f3cc89c7826019830d02fd34e51d Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 9 Jan 2026 11:03:11 -0500 Subject: [PATCH 1/2] feat(sabi): build rapidsmpf as limited API wheel --- .github/workflows/pr.yaml | 8 +++++++ ci/build_wheel_librapidsmpf.sh | 7 ++++++- ci/build_wheel_rapidsmpf.sh | 14 ++++++++++++- ci/build_wheel_singlecomm.sh | 7 ++++++- ci/test_wheel.sh | 12 +++++++++-- ci/use_upstream_sabi_wheels.sh | 37 +++++++++++++++++++++++++++++++++ python/rapidsmpf/pyproject.toml | 4 ++++ 7 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 ci/use_upstream_sabi_wheels.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5b1868c02..6ec5fc729 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -84,6 +84,10 @@ jobs: package-name: rapidsmpf package-type: python sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN + # Build a wheel for each CUDA x ARCH x PY_VER (3.10 and 3.11) + # When we drop Python 3.10, this can be simplified to: + # matrix_filter: map(select(.PY_VER == "3.11")) | unique_by({CUDA_VER, ARCH}) + matrix_filter: map(select(.PY_VER == "3.11" or .PY_VER == "3.10")) | unique_by({CUDA_VER, ARCH, PY_VER}) wheel-build-rapidsmpf-singlecomm: needs: [checks, wheel-build-librapidsmpf] secrets: inherit @@ -95,6 +99,10 @@ jobs: package-name: rapidsmpf-singlecomm package-type: python sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN + # Build a wheel for each CUDA x ARCH x PY_VER (3.10 and 3.11) + # When we drop Python 3.10, this can be simplified to: + # matrix_filter: map(select(.PY_VER == "3.11")) | unique_by({CUDA_VER, ARCH}) + matrix_filter: map(select(.PY_VER == "3.11" or .PY_VER == "3.10")) | unique_by({CUDA_VER, ARCH, PY_VER}) wheel-test: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python needs: [changed-files, wheel-build-rapidsmpf] diff --git a/ci/build_wheel_librapidsmpf.sh b/ci/build_wheel_librapidsmpf.sh index b08b38ea9..ed7225a8e 100755 --- a/ci/build_wheel_librapidsmpf.sh +++ b/ci/build_wheel_librapidsmpf.sh @@ -1,11 +1,16 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail source rapids-init-pip +# Only use stable ABI package for Python >= 3.11 +if [[ "${RAPIDS_PY_VERSION}" != "3.10" ]]; then + source ./ci/use_upstream_sabi_wheels.sh +fi + package_name="librapidsmpf" package_dir="python/librapidsmpf" diff --git a/ci/build_wheel_rapidsmpf.sh b/ci/build_wheel_rapidsmpf.sh index 6bf4f1a2f..953246ca0 100755 --- a/ci/build_wheel_rapidsmpf.sh +++ b/ci/build_wheel_rapidsmpf.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -7,6 +7,12 @@ set -euo pipefail package_name="rapidsmpf" package_dir="python/rapidsmpf" + +# Only use stable ABI package for Python >= 3.11 +if [[ "${RAPIDS_PY_VERSION}" != "3.10" ]]; then + source ./ci/use_upstream_sabi_wheels.sh +fi + source rapids-init-pip RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" @@ -53,3 +59,9 @@ python -m auditwheel repair \ ${package_dir}/dist/* ./ci/validate_wheel.sh "${package_dir}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" + +# Only use stable ABI package naming for Python >= 3.11 +if [[ "${RAPIDS_PY_VERSION}" != "3.10" ]]; then + RAPIDS_PACKAGE_NAME="$(rapids-package-name wheel_python rapidsmpf --stable --cuda)" + export RAPIDS_PACKAGE_NAME +fi diff --git a/ci/build_wheel_singlecomm.sh b/ci/build_wheel_singlecomm.sh index 682f8d154..3c3f77703 100755 --- a/ci/build_wheel_singlecomm.sh +++ b/ci/build_wheel_singlecomm.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # Tests building without MPI and without UCXX. This script only ensures the build @@ -10,6 +10,11 @@ set -euo pipefail source rapids-init-pip +# Only use stable ABI package for Python >= 3.11 +if [[ "${RAPIDS_PY_VERSION}" != "3.10" ]]; then + source ./ci/use_upstream_sabi_wheels.sh +fi + package_name="librapidsmpf" package_dir="python/librapidsmpf" package_name_py="rapidsmpf" diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index c4871ca81..6eb24e11a 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -eou pipefail @@ -7,8 +7,16 @@ set -eou pipefail source rapids-init-pip RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" + + +if [[ "${RAPIDS_PY_VERSION}" != "3.10" ]]; then + PYTHON_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" rapidsmpf --stable --cuda "$RAPIDS_CUDA_VERSION")") + source ./ci/use_upstream_sabi_wheels.sh +else + PYTHON_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="rapidsmpf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) +fi + CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="librapidsmpf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) -PYTHON_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="rapidsmpf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) # echo to expand wildcard before adding '[extra]' requires for pip rapids-pip-retry install \ diff --git a/ci/use_upstream_sabi_wheels.sh b/ci/use_upstream_sabi_wheels.sh new file mode 100644 index 000000000..264f57edd --- /dev/null +++ b/ci/use_upstream_sabi_wheels.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# initialize PIP_CONSTRAINT +source rapids-init-pip + +RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}") + +if [[ "${RAPIDS_PY_VERSION}" != "3.10" ]]; then + +# download wheels, store the directories holding them in variables +LIBRMM_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-artifact rmm 2184 cpp wheel +) +RMM_WHEELHOUSE=$( + rapids-get-pr-artifact rmm 2184 python wheel --stable +) +LIBCUDF_WHEELHOUSE=$( + RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-artifact cudf 20974 cpp wheel +) +PYLIBCUDF_WHEELHOUSE=$( + rapids-get-pr-artifact cudf 20974 python wheel --stable --pkg_name pylibcudf +) +CUDF_WHEELHOUSE=$( + rapids-get-pr-artifact cudf 20974 python wheel --stable +) + +cat > "${PIP_CONSTRAINT}" < Date: Fri, 9 Jan 2026 11:04:23 -0500 Subject: [PATCH 2/2] chore(ci): disable non-wheel CI --- .github/workflows/pr.yaml | 192 +++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6ec5fc729..aa80b0635 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,18 +14,18 @@ jobs: needs: - changed-files - checks - - conda-cpp-build - - conda-cpp-linters - - conda-cpp-tests - - conda-cpp-memcheck - - conda-python-build - - conda-python-tests - - docs-build + # - conda-cpp-build + # - conda-cpp-linters + # - conda-cpp-tests + # - conda-cpp-memcheck + # - conda-python-build + # - conda-python-tests + # - docs-build - wheel-build-librapidsmpf - wheel-build-rapidsmpf - wheel-build-rapidsmpf-singlecomm - wheel-test - - devcontainer + # - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() @@ -113,91 +113,91 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" script: ci/test_wheel.sh sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main - with: - build_type: pull-request - node_type: cpu8 - script: ci/build_cpp.sh - sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN - conda-cpp-linters: - secrets: inherit - needs: checks - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - with: - build_type: pull-request - script: "ci/cpp_linters.sh" - node_type: "cpu16" - sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN - conda-cpp-tests: - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - needs: [changed-files, conda-cpp-build] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main - with: - build_type: pull-request - container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" - script: ci/test_cpp.sh - sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN - conda-cpp-memcheck: - secrets: inherit - needs: conda-cpp-build - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - with: - build_type: pull-request - script: "ci/test_cpp_memcheck.sh" - node_type: "gpu-l4-latest-1" - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main - with: - build_type: pull-request - script: ci/build_python.sh - conda-python-tests: - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - needs: [changed-files, conda-python-build] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main - with: - build_type: pull-request - container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" - run_codecov: false - script: ci/test_python.sh - sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - with: - build_type: pull-request - node_type: "cpu8" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.04-latest" - script: "ci/build_docs.sh" - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@main - with: - arch: '["amd64", "arm64"]' - cuda: '["13.1"]' - python_package_manager: '["conda", "pip"]' - node_type: "cpu8" - rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN - env: | - SCCACHE_DIST_MAX_RETRIES=inf - SCCACHE_SERVER_LOG=sccache=debug - SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 - build_command: | - sccache --zero-stats; - build-all \ - -j0 \ - --verbose \ - -DBUILD_TESTS=OFF \ - -DBUILD_BENCHMARKS=ON \ - -DBUILD_NUMA_SUPPORT=OFF \ - 2>&1 | tee telemetry-artifacts/build.log; - sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt; + # conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@main + # with: + # build_type: pull-request + # node_type: cpu8 + # script: ci/build_cpp.sh + # sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN + # conda-cpp-linters: + # secrets: inherit + # needs: checks + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + # with: + # build_type: pull-request + # script: "ci/cpp_linters.sh" + # node_type: "cpu16" + # sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN + # conda-cpp-tests: + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + # needs: [changed-files, conda-cpp-build] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main + # with: + # build_type: pull-request + # container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" + # script: ci/test_cpp.sh + # sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN + # conda-cpp-memcheck: + # secrets: inherit + # needs: conda-cpp-build + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + # with: + # build_type: pull-request + # script: "ci/test_cpp_memcheck.sh" + # node_type: "gpu-l4-latest-1" + # conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@main + # with: + # build_type: pull-request + # script: ci/build_python.sh + # conda-python-tests: + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # needs: [changed-files, conda-python-build] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main + # with: + # build_type: pull-request + # container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" + # run_codecov: false + # script: ci/test_python.sh + # sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + # with: + # build_type: pull-request + # node_type: "cpu8" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:26.04-latest" + # script: "ci/build_docs.sh" + # devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@main + # with: + # arch: '["amd64", "arm64"]' + # cuda: '["13.1"]' + # python_package_manager: '["conda", "pip"]' + # node_type: "cpu8" + # rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN + # env: | + # SCCACHE_DIST_MAX_RETRIES=inf + # SCCACHE_SERVER_LOG=sccache=debug + # SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false + # SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 + # build_command: | + # sccache --zero-stats; + # build-all \ + # -j0 \ + # --verbose \ + # -DBUILD_TESTS=OFF \ + # -DBUILD_BENCHMARKS=ON \ + # -DBUILD_NUMA_SUPPORT=OFF \ + # 2>&1 | tee telemetry-artifacts/build.log; + # sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt;