diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index f8365c1dc5b..72b683fc459 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -13,14 +13,24 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") +# generate constraints (possibly pinning to oldest support versions of dependencies) +rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" + rapids-logger "Install libcudf, pylibcudf and cudf_polars" + +# notes: +# +# * echo to expand wildcard before adding `[test]` requires for pip +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI +# rapids-pip-retry install \ -v \ + --constraint "${PIP_CONSTRAINT}" \ "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" - TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")') rapids-logger "Clone polars to ${TAG}" git clone https://github.com/pola-rs/polars.git --branch "${TAG}" --depth 1 @@ -55,7 +65,17 @@ sed -i 's/^pandas-stubs/pandas-stubs<3/' polars/py-polars/requirements-dev.txt # existing dependency to rewrite. echo "pyparsing>=3.0.0,<3.3.0" >> polars/py-polars/requirements-dev.txt -rapids-pip-retry install -r polars/py-polars/requirements-dev.txt -r polars/py-polars/requirements-ci.txt +# notes: +# +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI +# +rapids-pip-retry install \ + -v \ + --prefer-binary \ + --constraint "${PIP_CONSTRAINT}" \ + -r polars/py-polars/requirements-dev.txt \ + -r polars/py-polars/requirements-ci.txt # shellcheck disable=SC2317 function set_exitcode() diff --git a/ci/test_cudf_polars_with_rapidsmpf.sh b/ci/test_cudf_polars_with_rapidsmpf.sh index 6e9ad45afe3..5072373e536 100755 --- a/ci/test_cudf_polars_with_rapidsmpf.sh +++ b/ci/test_cudf_polars_with_rapidsmpf.sh @@ -15,8 +15,18 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel rapids-logger "Installing cudf_polars and its dependencies (including rapidsmpf)" +# generate constraints (possibly pinning to oldest support versions of dependencies) +rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" + +# notes: +# +# * echo to expand wildcard before adding `[test]` requires for pip +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI +# rapids-pip-retry install \ -v \ + --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,rapidsmpf]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index a890c11f88d..3c8e3b84a1f 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -16,7 +16,7 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel rapids-logger "Install pylibcudf and its basic dependencies in a virtual environment" # generate constraints (possibly pinning to oldest support versions of dependencies) -rapids-generate-pip-constraints py_test_cudf ./constraints.txt +rapids-generate-pip-constraints py_test_cudf "${PIP_CONSTRAINT}" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ @@ -25,9 +25,16 @@ mkdir -p "${RAPIDS_TESTS_DIR}" # To test pylibcudf without its optional dependencies, we create a virtual environment python -m venv env . env/bin/activate + +# notes: +# +# * echo to expand wildcard before adding `[test]` requires for pip +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI +# rapids-pip-retry install \ -v \ - --constraint ./constraints.txt \ + --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" @@ -48,12 +55,12 @@ rapids-logger "Install cudf, pylibcudf, and test requirements" # notes: # # * echo to expand wildcard before adding `[test]` requires for pip -# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is -# ignored if any other --constraint are passed via the CLI +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI # rapids-pip-retry install \ -v \ - --constraint ./constraints.txt \ + --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index abdaeebd165..b67f8fd9f7b 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -18,17 +18,17 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel rapids-logger "Installing cudf_polars and its dependencies" # generate constraints (possibly pinning to oldest support versions of dependencies) -rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt +rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" # notes: # -# * echo to expand wildcard before adding `[test,experimental]` requires for pip -# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is -# ignored if any other --constraint are passed via the CLI +# * echo to expand wildcard before adding `[test]` requires for pip +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI # rapids-pip-retry install \ -v \ - --constraint ./constraints.txt \ + --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 636ced077d7..8930d85eab3 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -17,17 +17,17 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel rapids-logger "Install dask_cudf, cudf, pylibcudf, and test requirements" # generate constraints (possibly pinning to oldest support versions of dependencies) -rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt +rapids-generate-pip-constraints py_test_dask_cudf "${PIP_CONSTRAINT}" # notes: # # * echo to expand wildcard before adding `[test]` requires for pip -# * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is -# ignored if any other --constraint are passed via the CLI +# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because +# that environment variable is ignored if any other --constraint are passed via the CLI # rapids-pip-retry install \ -v \ - --constraint ./constraints.txt \ + --prefer-binary \ --constraint "${PIP_CONSTRAINT}" \ "$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${DASK_CUDF_WHEELHOUSE}"/dask_cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 65a20c827ed..67f921d9a67 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -129,7 +129,6 @@ outputs: run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - cuda-nvrtc - - libnvjitlink >=${{ cuda_version }} - if: linux and x86_64 then: - libcufile @@ -154,7 +153,6 @@ outputs: - libcufile - libcurand - libkvikio - - libnvjitlink - librdkafka - libzlib - librmm diff --git a/dependencies.yaml b/dependencies.yaml index a241b951728..4c43cf6d1d4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -204,6 +204,7 @@ files: table: project.optional-dependencies key: test includes: + - cuda_version - test_python_common - test_python_cudf_common - test_python_cudf @@ -352,6 +353,7 @@ files: table: project.optional-dependencies key: test includes: + - cuda_version - depends_on_dask_cuda - numpy_run - test_python_common @@ -390,6 +392,7 @@ files: table: project.optional-dependencies key: test includes: + - cuda_version - depends_on_dask_cuda - test_python_common - test_python_cudf_common @@ -563,6 +566,8 @@ dependencies: - output_types: [requirements, pyproject] packages: - pyarrow>=19.0.0 + # 'cuda_version' intentionally does not contain fallback entries... we want + # a loud error if an unsupported 'cuda' value is passed cuda_version: specific: - output_types: conda @@ -591,6 +596,51 @@ dependencies: cuda: "13.1" packages: - cuda-version=13.1 + - output_types: requirements + matrices: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) + - matrix: + use_cuda_wheels: "false" + packages: + - matrix: + arch: aarch64 + cuda: "12.2" + use_cuda_wheels: "true" + packages: + # some components (like nvidia-cublas-cu12 and nvidia-cuda-nvcc-cu12) didn't have + # aarch64 wheels until CTK 12.3, so allow a slightly looser bound here + - cuda-toolkit>=12.2,<12.4 + - matrix: + cuda: "12.2" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.2.* + - matrix: + cuda: "12.5" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.5.* + - matrix: + cuda: "12.8" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.8.* + - matrix: + cuda: "12.9" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.9.* + - matrix: + cuda: "13.0" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==13.0.* + - matrix: + cuda: "13.1" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==13.1.* cuda: common: - output_types: [conda] @@ -604,14 +654,30 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) - matrix: - cuda: "12.*" + use_cuda_wheels: "false" packages: - - nvidia-nvjitlink-cu12>=12.9 - # fallback to CUDA 13 versions if 'cuda' is '13.*' or not provided + # We always want nvJitLink >= whatever minor version was available in the build environment + # nvJitLink tends to follow the CTK's minor versions, but check at https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html + # + # ref: https://github.com/rapidsai/cudf/pull/20873 + - matrix: + cuda: "12.9" + use_cuda_wheels: "true" + packages: + - nvidia-nvjitlink-cu12>=12.9,<13 + - matrix: + cuda: "13.*" + use_cuda_wheels: "true" + packages: + - &nvjitlink_cu13 nvidia-nvjitlink>=13.0,<14 + # if no matching matrix selectors passed, list the CUDA 13 requirement + # (just as a source of documentation, as this populates pyproject.toml in source control) - matrix: packages: - - nvidia-nvjitlink>=13.0 + - *nvjitlink_cu13 develop: common: - output_types: [conda, requirements] @@ -725,14 +791,21 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) + - matrix: + use_cuda_wheels: "false" + packages: - matrix: cuda: "12.*" cuda_suffixed: "true" + use_cuda_wheels: "true" packages: - cuda-toolkit[nvcc,nvrtc]==12.* - matrix: cuda: "13.*" cuda_suffixed: "true" + use_cuda_wheels: "true" packages: - cuda-toolkit[nvcc,nvrtc]==13.* - {matrix: null, packages: []} diff --git a/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml b/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml index db3c55948c6..233c2e5b998 100644 --- a/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml +++ b/python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml @@ -138,14 +138,12 @@ dependencies: - output_types: conda packages: - pre-commit + # 'cuda_version' intentionally does not contain fallback entries... we want + # a loud error if an unsupported 'cuda' value is passed cuda_version: specific: - output_types: conda matrices: - - matrix: - cuda: "12.0" - packages: - - cuda-version=12.0 - matrix: cuda: "12.2" packages: @@ -170,6 +168,51 @@ dependencies: cuda: "13.1" packages: - cuda-version=13.1 + - output_types: requirements + matrices: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) + - matrix: + use_cuda_wheels: "false" + packages: + - matrix: + arch: aarch64 + cuda: "12.2" + use_cuda_wheels: "true" + packages: + # some components (like nvidia-cublas-cu12 and nvidia-cuda-nvcc-cu12) didn't have + # aarch64 wheels until CTK 12.3, so allow a slightly looser bound here + - cuda-toolkit>=12.2,<12.4 + - matrix: + cuda: "12.2" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.2.* + - matrix: + cuda: "12.5" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.5.* + - matrix: + cuda: "12.8" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.8.* + - matrix: + cuda: "12.9" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==12.9.* + - matrix: + cuda: "13.0" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==13.0.* + - matrix: + cuda: "13.1" + use_cuda_wheels: "true" + packages: + - cuda-toolkit==13.1.* py_version: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9810afa959c..29c34ddadd0 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -121,7 +121,7 @@ xfail_strict = true [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=3.30.4", "cython>=3.2.2", diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 173b3e36d24..f3d9ecd58be 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -73,6 +73,6 @@ requires = [ "librmm==26.4.*,>=0.0.0a0", "ninja", "nvidia-libnvcomp==5.1.0.21", - "nvidia-nvjitlink>=13.0", + "nvidia-nvjitlink>=13.0,<14", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 79cfa5c9e14..d3ecdb6fa4d 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -118,7 +118,7 @@ xfail_strict = true [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" requires = [ "cmake>=3.30.4", "cython>=3.2.2",