diff --git a/.githash b/.githash index 78440496..a82e977a 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -5f235a255f7b040ede53f151d0652d6352e087e9 +b1468726ba4827ab3e5ebb6e96ad94b9df78aa46 diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 1857b47f..d933f948 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -41,50 +41,29 @@ jobs: # runs-on: [self-hosted, linux, x64] runs-on: ubuntu-latest container: - # need nvcc - image: nvidia/cuda:12.4.0-devel-ubuntu22.04 - # options: --gpus all + image: ubuntu:22.04 defaults: run: shell: bash env: LEGATE_AUTO_CONFIG: 0 + NO_CUDA: ON steps: - name: Check out the repo uses: actions/checkout@v4 - name: bash run: source /etc/profile - - name: Get package spec - id: pkg - run: | - if [[ -n "${{ inputs.tag }}" ]]; then - echo "ref=${{ inputs.tag }}" >> $GITHUB_OUTPUT - echo "name=${{ inputs.tag }}" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_type }}" == "tag" ]]; then - echo "ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - elif [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "ref=${{ github.sha }}" >> $GITHUB_OUTPUT - echo "name=pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_type }}" == "branch" ]]; then - echo "ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - else - echo "ref=${{ github.sha }}" >> $GITHUB_OUTPUT - echo "name=dev" >> $GITHUB_OUTPUT - fi - VERSION=$(grep "^version = " Project.toml | cut -d'"' -f2) - echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Install basic utils + run: | + apt-get update && apt-get install -y wget curl git build-essential + curl -fsSL https://deb.nodesource.com/setup_20.x | bash - + apt-get install -y nodejs - name: Setup Julia uses: julia-actions/setup-julia@v2 with: version: '1.11' - - - name: Install basic utils - run: apt-get update && apt-get install -y wget curl git build-essential - - name: Install CMake 3.30.7 run: | @@ -98,7 +77,7 @@ jobs: id: julia-cache uses: julia-actions/cache@v2 with: - cache-name: julia-developer-ci-test-new + cache-name: julia-developer-ci - name: Cleanup old cuNumeric and Legate run: | julia --color=yes -e 'using Pkg; Pkg.rm("cuNumeric")' || true # ensure clean env @@ -113,16 +92,15 @@ jobs: using LegatePreferences; LegatePreferences.use_developer_mode(); Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/Legate.jl", rev = "main")) ' - julia --color=yes -e 'using Pkg; Pkg.build("Legate")' julia --color=yes -e ' using Pkg; - Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/cuNumeric.jl", rev = "${{ steps.pkg.outputs.ref }}", subdir = "lib/CNPreferences")) + Pkg.develop(PackageSpec(path = "lib/CNPreferences")) using CNPreferences; CNPreferences.use_developer_mode(); - Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/cuNumeric.jl", rev = "${{ steps.pkg.outputs.ref }}")) + Pkg.develop(PackageSpec(path = ".")) ' julia --color=yes -e 'using Pkg; Pkg.build("cuNumeric")' - name: Perform Test run: | - julia --color=yes -e 'using Pkg; Pkg.test("cuNumeric")' + GPUTESTS=0 julia --color=yes -e 'using Pkg; Pkg.test("cuNumeric")' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e8f82d94..45540153 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -66,22 +66,6 @@ jobs: julia --color=yes -e 'using Pkg; Pkg.rm("cuNumeric")' || true # ensure clean env julia --color=yes -e 'using Pkg; Pkg.rm("Legate")' || true # ensure clean env - - name: Setup CUDA runtime version - run: | - julia --color=yes --project=docs -e ' - using Pkg - Pkg.add("CUDA") - using CUDA - CUDA.set_runtime_version!(v"12.4") - ' - - name: Set LD_LIBRARY_PATH for CUDA_Driver_jll - run: | - echo "LD_LIBRARY_PATH=$(julia -e 'using Pkg; \ - Pkg.add(name = "CUDA_Driver_jll", version = "0.12.1"); \ - using CUDA_Driver_jll; \ - print(joinpath(CUDA_Driver_jll.artifact_dir, "lib"))' \ - ):$LD_LIBRARY_PATH" >> $GITHUB_ENV - - name: Clone cuNumeric at ref run: | REF=${{ steps.pkg.outputs.ref }} @@ -104,7 +88,6 @@ jobs: run: | julia --color=yes --project=docs -e ' using Pkg - Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/Legate.jl", rev = "main", subdir = "lib/LegatePreferences")) Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/Legate.jl", rev = "main")) Pkg.develop(path = joinpath(homedir(), ".julia", "dev", "cuNumeric/lib/CNPreferences")) Pkg.develop(path = joinpath(homedir(), ".julia", "dev", "cuNumeric")) diff --git a/Project.toml b/Project.toml index eaeb1779..72c7ef19 100644 --- a/Project.toml +++ b/Project.toml @@ -4,10 +4,6 @@ version = "0.1.0" [deps] CNPreferences = "3e078157-ea10-49d5-bf32-908f777cd46f" -CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" -CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc" -CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -CUTENSOR_jll = "35b6c64b-1ee1-5834-92a3-3f624899209a" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" CxxWrap = "1f15a43c-97ca-5a2a-ae31-89f07a497df4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" @@ -18,7 +14,6 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" -OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -29,14 +24,20 @@ cunumeric_jl_wrapper_jll = "49048992-29d2-5fd1-994f-9cecf112d624" cupynumeric_jll = "2862d674-414d-5b0b-a494-b21f8deca547" libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7" +[weakdeps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + [compat] -CUDA = "5.9" CNPreferences = "0.1.2" CxxWrap = "0.17" Legate = "0.1.0" LegatePreferences = "0.1.5" MacroTools = "0.5.16" +OpenBLAS32_jll = "0.3" StatsBase = "0.34" -cunumeric_jl_wrapper_jll = "25.10.0" -cupynumeric_jll = "25.10.0" +cunumeric_jl_wrapper_jll = "25.10.1" +cupynumeric_jll = "25.10.1" julia = "1.10" + +[ext] +CUDAExt = "CUDA" diff --git a/deps/build.jl b/deps/build.jl index 1196123d..431f4565 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -114,7 +114,7 @@ function build_jlcxxwrap(repo_root) end function build_cpp_wrapper( - repo_root, cupynumeric_loc, legate_loc, blas_lib, install_root + repo_root, cupynumeric_loc, legate_loc, blas_loc, install_root ) @info "libcunumeric_jl_wrapper: Building C++ Wrapper Library" if isdir(install_root) @@ -126,7 +126,7 @@ function build_cpp_wrapper( build_cpp_wrapper = joinpath(repo_root, "scripts/build_cpp_wrapper.sh") nthreads = Threads.nthreads() - bld_command = `$build_cpp_wrapper $repo_root $cupynumeric_loc $legate_loc $blas_lib $install_root $nthreads` + bld_command = `$build_cpp_wrapper $repo_root $cupynumeric_loc $legate_loc $blas_loc $install_root $nthreads` # write out a bash script for debugging cmd_str = join(bld_command.exec, " ") @@ -197,7 +197,7 @@ function build(mode) ) end build_cpp_wrapper( - pkg_root, cupynumeric_root, up_dir(legate_lib), blas_lib, + pkg_root, cupynumeric_root, up_dir(legate_lib), up_dir(blas_lib), install_lib, ) end diff --git a/ext/CUDAExt/CUDAExt.jl b/ext/CUDAExt/CUDAExt.jl new file mode 100644 index 00000000..5bbe5e4e --- /dev/null +++ b/ext/CUDAExt/CUDAExt.jl @@ -0,0 +1,20 @@ +module CUDAExt + +using Random +using CUDA +using cuNumeric + +include("cuda.jl") + +function __init__() + if CUDA.functional() + # in cuda.jl to notify /wrapper/src/cuda.cpp about CUDA.jl kernel state size + cuNumeric.set_kernel_state_size(); + # in /wrapper/src/cuda.cpp + cuNumeric.register_tasks(); + else + @warn "CUDA.jl is not functional; skipping CUDA kernel registration." + end +end + +end # module CUDAExt \ No newline at end of file diff --git a/src/cuda.jl b/ext/CUDAExt/cuda.jl similarity index 99% rename from src/cuda.jl rename to ext/CUDAExt/cuda.jl index b6f3ef3d..5c64904a 100644 --- a/src/cuda.jl +++ b/ext/CUDAExt/cuda.jl @@ -1,9 +1,8 @@ -using CUDA -using Random +export @cuda_task, @launch const KERNEL_OFFSET = sizeof(CUDA.KernelState) -# cuNumeric.jl init will call this +# __init__ will call this # kernel_state is the first ARG in the generated PTX in CUDA.jl function set_kernel_state_size() cuNumeric.register_kernel_state_size(UInt64(KERNEL_OFFSET)) diff --git a/scripts/build_cpp_wrapper.sh b/scripts/build_cpp_wrapper.sh index 13ad77f5..6466d787 100755 --- a/scripts/build_cpp_wrapper.sh +++ b/scripts/build_cpp_wrapper.sh @@ -2,13 +2,13 @@ set -e # Check if exactly one argument is provided if [[ $# -ne 6 ]]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi CUNUMERICJL_ROOT_DIR=$1 # this is the repo root of cunumeric.jl CUPYNUMERIC_ROOT_DIR=$2 LEGATE_ROOT_DIR=$3 -BLAS_LIB_DIR=$4 +BLAS_ROOT_DIR=$4 INSTALL_DIR=$5 NTHREADS=$6 @@ -42,10 +42,14 @@ fi echo $LEGATE_ROOT_DIR +# Default to OFF (CUDA support enabled), but allow override via environment variable +NO_CUDA=${NO_CUDA:-OFF} + if [[ ! -f "$BUILD_DIR/CMakeCache.txt" ]]; then echo "Configuring project..." cmake -S "$CUNUMERIC_WRAPPER_SOURCE" -B "$BUILD_DIR" \ -D BINARYBUILDER=OFF \ + -D NOCUDA=$NO_CUDA \ -D CMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ -D CMAKE_PREFIX_PATH="$CUPYNUMERIC_ROOT_DIR;$LEGATE_ROOT_DIR;" \ -D CUPYNUMERIC_PATH="$CUPYNUMERIC_ROOT_DIR" \ diff --git a/src/cuNumeric.jl b/src/cuNumeric.jl index b9abd85c..77d9a9f3 100644 --- a/src/cuNumeric.jl +++ b/src/cuNumeric.jl @@ -40,39 +40,73 @@ const SUPPORTED_TYPES = Union{SUPPORTED_INT_TYPES,SUPPORTED_FLOAT_TYPES,Bool} #* # const MAX_DIM = 6 # idk what we compiled? include("utilities/preference.jl") -find_preferences() - -const BLAS_LIB = load_preference(CNPreferences, "BLAS_LIB", nothing) -const CUTENSOR_LIB = load_preference(CNPreferences, "CUTENSOR_LIB", nothing) -const TBLIS_LIB = load_preference(CNPreferences, "TBLIS_LIB", nothing) -const CUPYNUMERIC_LIB = load_preference(CNPreferences, "CUPYNUMERIC_LIB", nothing) -const CUNUMERIC_WRAPPER_LIB = load_preference(CNPreferences, "CUNUMERIC_WRAPPER_LIB", nothing) - -libnda = joinpath(CUNUMERIC_WRAPPER_LIB, "libcunumeric_c_wrapper.so") -libpath = joinpath(CUNUMERIC_WRAPPER_LIB, "libcunumeric_jl_wrapper.so") -if !isfile(libpath) - error("Developer mode: You need to call Pkg.build()") -end - -function preload_libs() - libs = [ - joinpath(OpenBLAS32_jll.artifact_dir, "lib", "libopenblas.so"), # required for libcupynumeric.so - joinpath(TBLIS_LIB, "libtblis.so"), - joinpath(CUPYNUMERIC_LIB, "libcupynumeric.so"), - ] - if HAS_CUDA - push!(libs, joinpath(CUTENSOR_LIB, "libcutensor.so")) +# Sets the LEGATE_LIB_PATH and WRAPPER_LIB_PATH preferences based on mode +# This will also include the relevant JLLs if necessary. +@static if CNPreferences.MODE == "jll" + using cupynumeric_jll, cunumeric_jl_wrapper_jll + find_paths( + CNPreferences.MODE; + cupynumeric_jll_module=cupynumeric_jll, + cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll, + ) +elseif CNPreferences.MODE == "developer" + use_cupynumeric_jll = load_preference(CNPreferences, "legate_use_jll", true) + if use_cupynumeric_jll + using cupynumeric_jll + find_paths( + CNPreferences.MODE; + cupynumeric_jll_module=cupynumeric_jll, + cupynumeric_jll_wrapper_module=nothing, + ) + else + find_paths(CNPreferences.MODE) end +elseif CNPreferences.MODE == "conda" + using cunumeric_jl_wrapper_jll + find_paths( + CNPreferences.MODE, + cupynumeric_jll_module=nothing, + cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll, + ) +else + error( + "cuNumeric.jl: Unknown mode $(CNPreferences.MODE). Must be one of 'jll', 'developer', or 'conda'." + ) +end - for lib in libs - Libdl.dlopen(lib, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) +const CUPYNUMERIC_LIBDIR = load_preference(CNPreferences, "CUPYNUMERIC_LIBDIR", nothing) +const CUPYNUMERIC_WRAPPER_LIBDIR = load_preference( + CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR", nothing +) + +const libnda = joinpath(CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_c_wrapper.so") +const CUPYNUMERIC_WRAPPER_LIB_PATH = joinpath( + CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_jl_wrapper.so" +) +const CUPYNUMERIC_LIB_PATH = joinpath(CUPYNUMERIC_LIBDIR, "libcupynumeric.so") + +(isnothing(CUPYNUMERIC_LIBDIR) || isnothing(CUPYNUMERIC_WRAPPER_LIBDIR)) && error( + "cuNumeric.jl: CUPYNUMERIC_LIBDIR or CUPYNUMERIC_WRAPPER_LIBDIR preference not set. Check LocalPreferences.toml" +) + +if !isfile(CUPYNUMERIC_WRAPPER_LIB_PATH) + # Print build error logs if available + deps = joinpath(dirname(@__DIR__), "deps") + for errfile in ["cpp_wrapper.err", "libcxxwrap.err"] + errpath = joinpath(deps, errfile) + if isfile(errpath) + println("\n=== Contents of $errfile ===") + println(read(errpath, String)) + println("=== End of $errfile ===\n") + end end + error( + "Developer mode: You need to call Pkg.build(). Library $CUPYNUMERIC_WRAPPER_LIB_PATH not found." + ) end -preload_libs() # for precompilation - -@wrapmodule(() -> libpath) +@wrapmodule(() -> CUPYNUMERIC_WRAPPER_LIB_PATH) # custom GC include("memory.jl") @@ -93,11 +127,6 @@ include("ndarray/binary.jl") # scoping macro include("scoping.jl") -# # Custom CUDA.jl kernel integration -if HAS_CUDA - include("cuda.jl") -end - # # Utilities include("utilities/version.jl") include("util.jl") @@ -122,20 +151,6 @@ end global cuNumeric_config_str::String = "" -function cunumeric_setup(AA::ArgcArgv) - Base.atexit(my_on_exit) - - cuNumeric.initialize_cunumeric(AA.argc, getargv(AA)) - if HAS_CUDA - # in /src/cuda.jl to notify /wrapper/src/cuda.cpp about CUDA.jl kernel state size - cuNumeric.set_kernel_state_size(); - # in /wrapper/src/cuda.cpp - cuNumeric.register_tasks(); - end - # setup /src/memory.jl - cuNumeric.init_gc!() -end - @doc""" versioninfo() @@ -149,12 +164,21 @@ end # Runtime initilization function __init__() CNPreferences.check_unchanged() - preload_libs() + + Libdl.dlopen(CUPYNUMERIC_LIB_PATH, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) + Libdl.dlopen(CUPYNUMERIC_WRAPPER_LIB_PATH, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) + @initcxx AA = ArgcArgv([Base.julia_cmd()[1]]) global cuNumeric_config_str = version_config_setup() - cunumeric_setup(AA) + + cuNumeric.initialize_cunumeric(AA.argc, getargv(AA)) + + # setup /src/memory.jl + cuNumeric.init_gc!() + + Base.atexit(my_on_exit) end end #module cuNumeric diff --git a/src/utilities/depends.jl b/src/utilities/depends.jl index 7218e095..984ab504 100644 --- a/src/utilities/depends.jl +++ b/src/utilities/depends.jl @@ -1,8 +1,7 @@ using Preferences -using CNPreferences: CNPreferences +using CNPreferences +import LegatePreferences using Legate -using OpenSSL_jll -using OpenBLAS32_jll using Libdl using CxxWrap using Pkg @@ -10,7 +9,6 @@ using TOML using cupynumeric_jll using cunumeric_jl_wrapper_jll -using CUTENSOR_jll import Base: axes, convert, copy, copyto!, inv, isfinite, sqrt, -, +, *, ==, !=, isapprox, read, view, maximum, minimum, prod, sum, getindex, setindex!, diff --git a/src/utilities/preference.jl b/src/utilities/preference.jl index c289c310..212c0c48 100644 --- a/src/utilities/preference.jl +++ b/src/utilities/preference.jl @@ -51,54 +51,117 @@ function get_library_root(jll_module, env_var::String) end end -function find_preferences() - pkg_root = abspath(joinpath(@__DIR__, "../", "../")) - - blas_lib = get_library_root(OpenBLAS32_jll, "JULIA_OPENBLAS_PATH") - if HAS_CUDA - cutensor_lib = get_library_root(CUTENSOR_jll, "JULIA_CUTENSOR_PATH") - end +############################################# +# LOTS OF THIS LOGIC EXISTS IN LEGATE.JL TOO +# DEFINITELY DUPLICATED CODE +############################################# - cupynumeric_path = cupynumeric_jll.artifact_dir +function check_jll(m::Module) + if !m.is_available() + m_host_cuda = cupynumeric_jll.host_platform["cuda"] - mode = load_preference(CNPreferences, "cunumeric_mode", CNPreferences.MODE_JLL) + if (m_host_cuda == "none") + error( + "$(string(m)) installed but not available on this platform.\n $(string(cupynumeric_jll.host_platform))", + ) + end - # if developer mode - if mode == CNPreferences.MODE_JLL - cunumeric_wrapper_lib = joinpath(cunumeric_jl_wrapper_jll.artifact_dir, "lib") - elseif mode == CNPreferences.MODE_DEVELOPER - use_cupynumeric_jll = load_preference( - CNPreferences, "cunumeric_use_jll", true - ) - if use_cupynumeric_jll == false - cupynumeric_path = load_preference( - CNPreferences, "cunumeric_path", nothing + v_host_cuda = VersionNumber(m_host_cuda) + valid_cuda_version = Legate.MIN_CUDA_VERSION <= v_host_cuda <= Legate.MAX_CUDA_VERSION + if !valid_cuda_version + error( + "$(string(m)) installed but not available on this platform. Host CUDA ver: $(v_host_cuda) not in range supported by $(string(m)): $(MIN_CUDA_VERSION)-$(MAX_CUDA_VERSION).", ) - check_cupynumeric_install(cupynumeric_path) + else + error("$(string(m)) installed but not available on this platform. Unknown reason.") end - cunumeric_wrapper_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build", "lib") - # if conda - elseif mode == CNPreferences.MODE_CONDA - @warn "mode = conda may break. We are using a subset of libraries from conda." - conda_env = load_preference(CNPreferences, "cunumeric_conda_env", nothing) - check_cupynumeric_install(conda_env) - cupynumeric_path = conda_env - cutensor_lib = joinpath(conda_env, "lib") end +end - cupynumeric_lib = joinpath(cupynumeric_path, "lib") - if haskey(ENV, "JULIA_TBLIS_PATH") - tblis_lib = get(ENV, "JULIA_TBLIS_PATH", "0") +function find_paths( + mode::String; + cupynumeric_jll_module::Union{Module,Nothing}=nothing, + cupynumeric_jll_wrapper_module::Union{Module,Nothing}=nothing, +) + libcupynumeric_path, libcupynumeric_wrapper_path = cuNumeric._find_paths( + CNPreferences.to_mode(mode), cupynumeric_jll_module, cupynumeric_jll_wrapper_module + ) + set_preferences!(CNPreferences, "CUPYNUMERIC_LIBDIR" => libcupynumeric_path; force=true) + set_preferences!( + CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR" => libcupynumeric_wrapper_path; force=true + ) +end + +function _find_paths( + mode::CNPreferences.JLL, + cupynumeric_jll_module::Module, + cupynumeric_jll_wrapper_module::Module, +) + check_jll(cupynumeric_jll_module) + check_jll(cupynumeric_jll_wrapper_module) + legate_lib_dir = joinpath(cupynumeric_jll_module.artifact_dir, "lib") + legate_wrapper_libdir = joinpath(cupynumeric_jll_wrapper_module.artifact_dir, "lib") + return legate_lib_dir, legate_wrapper_libdir +end + +function _find_paths( + mode::CNPreferences.Developer, + cupynumeric_jll_module::Module, + cupynumeric_jll_wrapper_module::Nothing, +) + cupynumeric_path = "" + use_cupynumeric_jll = load_preference(CNPreferences, "cupynumeric_use_jll", true) + + if use_cupynumeric_jll == false + cupynumeric_path = load_preference(CNPreferences, "cupynumeric_path", nothing) + check_cupynumeric_install(cupynumeric_path) else - tblis_lib = cupynumeric_lib # cupynumeric libpath will by default contain tblis + check_jll(cupynumeric_jll_module) + cupynumeric_path = cupynumeric_jll.artifact_dir end - if HAS_CUDA - set_preferences!(CNPreferences, "CUTENSOR_LIB" => cutensor_lib; force=true) - end + pkg_root = abspath(joinpath(@__DIR__, "../", "../")) + wrapper_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build", "lib") + + return joinpath(cupynumeric_path, "lib"), wrapper_lib +end + +function _find_paths( + mode::CNPreferences.Conda, + cupynumeric_jll_module::Nothing, + cupynumeric_jll_wrapper_module::Module, +) + @warn "mode = conda may break. We are using a subset of libraries from conda." + + conda_env = load_preference(CNPreferences, "legate_conda_env", nothing) + isnothing(conda_env) && error( + "legate_conda_env preference must be set in LocalPreferences.toml when using conda mode" + ) + + check_legate_install(conda_env) + legate_path = conda_env + check_jll(cupynumeric_jll_wrapper_module) + legate_wrapper_lib = joinpath(cupynumeric_jll_wrapper_module.artifact_dir, "lib") - set_preferences!(CNPreferences, "BLAS_LIB" => blas_lib; force=true) - set_preferences!(CNPreferences, "TBLIS_LIB" => tblis_lib; force=true) - set_preferences!(CNPreferences, "CUPYNUMERIC_LIB" => cupynumeric_lib; force=true) - set_preferences!(CNPreferences, "CUNUMERIC_WRAPPER_LIB" => cunumeric_wrapper_lib; force=true) + return joinpath(legate_path, "lib"), legate_wrapper_lib end + +# MPI, NCCL etc are found by Legate.find_dependency_paths +const DEPS_MAP = Dict( + "CUTENSOR" => "libcutensor", + "BLAS" => "libopenblas", + "TBLIS" => "libtblis", +) +function find_dependency_paths(::Type{CNPreferences.JLL}) + results = Dict{String,String}() + + paths_to_search = copy(cupynumeric_jll.LIBPATH_list) + + for (name, lib) in DEPS_MAP + results[name] = dirname(Libdl.find_library(lib, paths_to_search)) + end + return results +end + +find_dependency_paths(::Type{CNPreferences.Developer}) = Dict{String,String}() +find_dependency_paths(::Type{CNPreferences.Conda}) = Dict{String,String}() diff --git a/src/utilities/version.jl b/src/utilities/version.jl index 9225f051..b3e2495e 100644 --- a/src/utilities/version.jl +++ b/src/utilities/version.jl @@ -39,7 +39,7 @@ function version_config_setup() name = get(project, "name", "unknown") version = get(project, "version", "unknown") uuid = get(project, "uuid", "unknown") - compiler = get_cxx_version(libpath) + compiler = get_cxx_version(CUPYNUMERIC_LIB_PATH) julia_ver = VERSION hostname = gethostname() @@ -47,23 +47,13 @@ function version_config_setup() liblegate = Legate.LEGATE_LIBDIR liblegatewrapper = Legate.LEGATE_WRAPPER_LIBDIR - if Legate.LegatePreferences.MODE == "jll" - other_dirs = Legate.find_dependency_paths(Legate.JLL()) - else - other_dirs = Dict( - "HDF5" => "unknown", - "MPI" => "unknown", - "NCCL" => "unknown", - "CUDA_DRIVER" => "unknown", - "CUDA_RUNTIME" => "unknown", - ) - end + other_dirs = Dict() - libblas = BLAS_LIB - libcutensor = CUTENSOR_LIB - libcupynumeric = CUPYNUMERIC_LIB - libtblis = TBLIS_LIB - libcunumericwrapper = CUNUMERIC_WRAPPER_LIB + cn_mode = CNPreferences.to_mode(CNPreferences.MODE) + legate_mode = LegatePreferences.to_mode(LegatePreferences.MODE) + dirs1 = cuNumeric.find_dependency_paths(typeof(cn_mode)) + dirs2 = Legate.find_dependency_paths(typeof(legate_mode)) + other_dirs = merge(dirs1, dirs2) str = """ ─────────────────────────────────────────────── @@ -77,22 +67,26 @@ function version_config_setup() Hostname: $hostname Julia Version: $julia_ver C++ Compiler: $compiler - CUDA Driver: $(other_dirs["CUDA_DRIVER"]) - CUDA Runtime: $(other_dirs["CUDA_RUNTIME"]) + CUDA Driver: $(get(other_dirs,"CUDA_DRIVER","unknown")) + CUDA Runtime: $(get(other_dirs,"CUDA_RUNTIME","unknown")) Library Paths: Legate: $liblegate - cuPyNumeric: $libcupynumeric - BLAS: $libblas - TBLIS: $libtblis - CUTENSOR: $libcutensor - NCCL: $(other_dirs["NCCL"]) - MPI: $(other_dirs["MPI"]) - HDF5: $(other_dirs["HDF5"]) + cuPyNumeric: $(CUPYNUMERIC_LIBDIR) + BLAS: $(get(other_dirs,"BLAS","unknown")) + TBLIS: $(get(other_dirs,"TBLIS","unknown")) + CUTENSOR: $(get(other_dirs,"CUTENSOR","unknown")) + NCCL: $(get(other_dirs,"NCCL","unknown")) + MPI: $(get(other_dirs,"MPI","unknown")) + HDF5: $(get(other_dirs,"HDF5","unknown")) Wrappers: - cuNumeric $libcunumericwrapper + cuNumeric $(CUPYNUMERIC_WRAPPER_LIBDIR) Legate $liblegatewrapper + + Modes: + cuNumeric: $(CNPreferences.MODE) + Legate: $(LegatePreferences.MODE) ─────────────────────────────────────────────── """ return str diff --git a/test/runtests.jl b/test/runtests.jl index 60b90c18..2f7e3d1e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,37 +18,18 @@ =# using Test -using cuNumeric - using LinearAlgebra -using CUDA -import CUDA: i32 - using Random import Random: rand -include("tests/util.jl") -include("tests/axpy.jl") -include("tests/axpy_advanced.jl") -include("tests/elementwise.jl") -include("tests/slicing.jl") -include("tests/gemm.jl") -include("tests/unary_tests.jl") -include("tests/binary_tests.jl") -include("tests/scoping.jl") -include("tests/scoping-advanced.jl") - -const VERBOSE = false - +const VERBOSE = get(ENV, "VERBOSE", "1") != "0" const run_gpu_tests = get(ENV, "GPUTESTS", "1") != "0" -const run_cuda_tests = run_gpu_tests && CUDA.functional() +@info "Run GPU Tests: $(run_gpu_tests)" -if VERBOSE - cuNumeric.versioninfo() -end - -if run_gpu_tests && VERBOSE - println(CUDA.versioninfo()) +if run_gpu_tests + using CUDA + import CUDA: i32 + VERBOSE && println(CUDA.versioninfo()) end if run_gpu_tests && !CUDA.functional() @@ -57,7 +38,25 @@ if run_gpu_tests && !CUDA.functional() ) end -@info "Run CUDA Tests: $(run_cuda_tests)" +using cuNumeric +VERBOSE && cuNumeric.versioninfo() + +# TODO +# After loading cuNumeric, we should verify that the Legate config has set a GPU device +# Right now, if you have a gpu device, but your LEGATE_CONFIG is cpu only, +# @cuda_task will not be defined and tests will fail confusingly. +# We should error out more gracefully in this situation. + +include("tests/util.jl") +include("tests/axpy.jl") +include("tests/axpy_advanced.jl") +include("tests/elementwise.jl") +include("tests/slicing.jl") +include("tests/gemm.jl") +include("tests/unary_tests.jl") +include("tests/binary_tests.jl") +include("tests/scoping.jl") +include("tests/scoping-advanced.jl") @testset verbose = true "AXPY" begin N = 100 @@ -398,7 +397,7 @@ end end end -if run_cuda_tests +if run_gpu_tests include("tests/cuda/vecadd.jl") @testset verbose = true "CUDA Tests" begin cuda_unaryop(rtol(Float32))