From d351813255a882bfe3a5ad13aaee264a146a1257 Mon Sep 17 00:00:00 2001 From: ejmeitz Date: Mon, 12 Jan 2026 15:59:40 -0600 Subject: [PATCH 01/17] update dep loading --- Project.toml | 11 +-- ext/CUDAExt/CUDAExt.jl | 20 +++++ {src => ext/CUDAExt}/cuda.jl | 5 +- src/cuNumeric.jl | 97 ++++++++++++------------ src/utilities/depends.jl | 6 +- src/utilities/preference.jl | 139 ++++++++++++++++++++++++----------- src/utilities/version.jl | 48 ++++++------ test/runtests.jl | 41 +++++------ 8 files changed, 216 insertions(+), 151 deletions(-) create mode 100644 ext/CUDAExt/CUDAExt.jl rename {src => ext/CUDAExt}/cuda.jl (99%) diff --git a/Project.toml b/Project.toml index eaeb1779..67cd775f 100644 --- a/Project.toml +++ b/Project.toml @@ -4,10 +4,6 @@ version = "0.1.0" [deps] CNPreferences = "3e078157-ea10-49d5-bf32-908f777cd46f" -CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" -CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc" -CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -CUTENSOR_jll = "35b6c64b-1ee1-5834-92a3-3f624899209a" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" CxxWrap = "1f15a43c-97ca-5a2a-ae31-89f07a497df4" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" @@ -17,8 +13,6 @@ LegatePreferences = "8028f36a-2b64-49e9-aa04-2d0933fd2ed9" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" -OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -30,13 +24,12 @@ cupynumeric_jll = "2862d674-414d-5b0b-a494-b21f8deca547" libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7" [compat] -CUDA = "5.9" CNPreferences = "0.1.2" CxxWrap = "0.17" Legate = "0.1.0" LegatePreferences = "0.1.5" MacroTools = "0.5.16" StatsBase = "0.34" -cunumeric_jl_wrapper_jll = "25.10.0" -cupynumeric_jll = "25.10.0" +cunumeric_jl_wrapper_jll = "25.10.1" +cupynumeric_jll = "25.10.1" julia = "1.10" diff --git a/ext/CUDAExt/CUDAExt.jl b/ext/CUDAExt/CUDAExt.jl new file mode 100644 index 00000000..5bbe5e4e --- /dev/null +++ b/ext/CUDAExt/CUDAExt.jl @@ -0,0 +1,20 @@ +module CUDAExt + +using Random +using CUDA +using cuNumeric + +include("cuda.jl") + +function __init__() + if CUDA.functional() + # in cuda.jl to notify /wrapper/src/cuda.cpp about CUDA.jl kernel state size + cuNumeric.set_kernel_state_size(); + # in /wrapper/src/cuda.cpp + cuNumeric.register_tasks(); + else + @warn "CUDA.jl is not functional; skipping CUDA kernel registration." + end +end + +end # module CUDAExt \ No newline at end of file diff --git a/src/cuda.jl b/ext/CUDAExt/cuda.jl similarity index 99% rename from src/cuda.jl rename to ext/CUDAExt/cuda.jl index b6f3ef3d..5c64904a 100644 --- a/src/cuda.jl +++ b/ext/CUDAExt/cuda.jl @@ -1,9 +1,8 @@ -using CUDA -using Random +export @cuda_task, @launch const KERNEL_OFFSET = sizeof(CUDA.KernelState) -# cuNumeric.jl init will call this +# __init__ will call this # kernel_state is the first ARG in the generated PTX in CUDA.jl function set_kernel_state_size() cuNumeric.register_kernel_state_size(UInt64(KERNEL_OFFSET)) diff --git a/src/cuNumeric.jl b/src/cuNumeric.jl index b9abd85c..715a4376 100644 --- a/src/cuNumeric.jl +++ b/src/cuNumeric.jl @@ -40,39 +40,55 @@ const SUPPORTED_TYPES = Union{SUPPORTED_INT_TYPES,SUPPORTED_FLOAT_TYPES,Bool} #* # const MAX_DIM = 6 # idk what we compiled? include("utilities/preference.jl") -find_preferences() -const BLAS_LIB = load_preference(CNPreferences, "BLAS_LIB", nothing) -const CUTENSOR_LIB = load_preference(CNPreferences, "CUTENSOR_LIB", nothing) -const TBLIS_LIB = load_preference(CNPreferences, "TBLIS_LIB", nothing) -const CUPYNUMERIC_LIB = load_preference(CNPreferences, "CUPYNUMERIC_LIB", nothing) -const CUNUMERIC_WRAPPER_LIB = load_preference(CNPreferences, "CUNUMERIC_WRAPPER_LIB", nothing) - -libnda = joinpath(CUNUMERIC_WRAPPER_LIB, "libcunumeric_c_wrapper.so") -libpath = joinpath(CUNUMERIC_WRAPPER_LIB, "libcunumeric_jl_wrapper.so") -if !isfile(libpath) - error("Developer mode: You need to call Pkg.build()") +# Sets the LEGATE_LIB_PATH and WRAPPER_LIB_PATH preferences based on mode +# This will also include the relevant JLLs if necessary. +@static if CNPreferences.MODE == "jll" + using cupynumeric_jll, cunumeric_jl_wrapper_jll + find_paths( + CNPreferences.MODE; + cupynumeric_jll_module=cupynumeric_jll, + cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll + ) +elseif CNPreferences.MODE == "developer" + use_cupynumeric_jll = load_preference(CNPreferences, "legate_use_jll", true) + if use_cupynumeric_jll + using cupynumeric_jll + find_paths( + CNPreferences.MODE; + cupynumeric_jll_module=cupynumeric_jll, + cupynumeric_jll_wrapper_module=nothing + ) + else + find_paths(CNPreferences.MODE) + end +elseif CNPreferences.MODE == "conda" + using cunumeric_jl_wrapper_jll + find_paths( + CNPreferences.MODE, + cupynumeric_jll_module=nothing, + cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll + ) +else + error("cuNumeric.jl: Unknown mode $(CNPreferences.MODE). Must be one of 'jll', 'developer', or 'conda'.") end -function preload_libs() - libs = [ - joinpath(OpenBLAS32_jll.artifact_dir, "lib", "libopenblas.so"), # required for libcupynumeric.so - joinpath(TBLIS_LIB, "libtblis.so"), - joinpath(CUPYNUMERIC_LIB, "libcupynumeric.so"), - ] - if HAS_CUDA - push!(libs, joinpath(CUTENSOR_LIB, "libcutensor.so")) - end +const CUPYNUMERIC_LIBDIR = load_preference(CNPreferences, "CUPYNUMERIC_LIBDIR", nothing) +const CUPYNUMERIC_WRAPPER_LIBDIR = load_preference(CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR", nothing) - for lib in libs - Libdl.dlopen(lib, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) - end +const libnda = joinpath(CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_c_wrapper.so") +const CUPYNUMERIC_WRAPPER_LIB_PATH = joinpath(CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_jl_wrapper.so") +const CUPYNUMERIC_LIB_PATH = joinpath(CUPYNUMERIC_LIBDIR, "libcupynumeric.so") + +(isnothing(CUPYNUMERIC_LIBDIR) || isnothing(CUPYNUMERIC_WRAPPER_LIBDIR)) && error("cuNumeric.jl: CUPYNUMERIC_LIBDIR or CUPYNUMERIC_WRAPPER_LIBDIR preference not set. Check LocalPreferences.toml") + +if !isfile(CUPYNUMERIC_WRAPPER_LIB_PATH) + error("Developer mode: You need to call Pkg.build()") end -preload_libs() # for precompilation -@wrapmodule(() -> libpath) +@wrapmodule(() -> CUPYNUMERIC_WRAPPER_LIB_PATH) # custom GC include("memory.jl") @@ -93,11 +109,6 @@ include("ndarray/binary.jl") # scoping macro include("scoping.jl") -# # Custom CUDA.jl kernel integration -if HAS_CUDA - include("cuda.jl") -end - # # Utilities include("utilities/version.jl") include("util.jl") @@ -122,19 +133,6 @@ end global cuNumeric_config_str::String = "" -function cunumeric_setup(AA::ArgcArgv) - Base.atexit(my_on_exit) - - cuNumeric.initialize_cunumeric(AA.argc, getargv(AA)) - if HAS_CUDA - # in /src/cuda.jl to notify /wrapper/src/cuda.cpp about CUDA.jl kernel state size - cuNumeric.set_kernel_state_size(); - # in /wrapper/src/cuda.cpp - cuNumeric.register_tasks(); - end - # setup /src/memory.jl - cuNumeric.init_gc!() -end @doc""" versioninfo() @@ -149,12 +147,21 @@ end # Runtime initilization function __init__() CNPreferences.check_unchanged() - preload_libs() + + Libdl.dlopen(CUPYNUMERIC_LIB_PATH, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) + Libdl.dlopen(CUPYNUMERIC_WRAPPER_LIB_PATH, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) + @initcxx AA = ArgcArgv([Base.julia_cmd()[1]]) global cuNumeric_config_str = version_config_setup() - cunumeric_setup(AA) + + cuNumeric.initialize_cunumeric(AA.argc, getargv(AA)) + + # setup /src/memory.jl + cuNumeric.init_gc!() + + Base.atexit(my_on_exit) end end #module cuNumeric diff --git a/src/utilities/depends.jl b/src/utilities/depends.jl index 7218e095..984ab504 100644 --- a/src/utilities/depends.jl +++ b/src/utilities/depends.jl @@ -1,8 +1,7 @@ using Preferences -using CNPreferences: CNPreferences +using CNPreferences +import LegatePreferences using Legate -using OpenSSL_jll -using OpenBLAS32_jll using Libdl using CxxWrap using Pkg @@ -10,7 +9,6 @@ using TOML using cupynumeric_jll using cunumeric_jl_wrapper_jll -using CUTENSOR_jll import Base: axes, convert, copy, copyto!, inv, isfinite, sqrt, -, +, *, ==, !=, isapprox, read, view, maximum, minimum, prod, sum, getindex, setindex!, diff --git a/src/utilities/preference.jl b/src/utilities/preference.jl index c289c310..249a8517 100644 --- a/src/utilities/preference.jl +++ b/src/utilities/preference.jl @@ -51,54 +51,111 @@ function get_library_root(jll_module, env_var::String) end end -function find_preferences() - pkg_root = abspath(joinpath(@__DIR__, "../", "../")) +############################################# +# LOTS OF THIS LOGIC EXISTS IN LEGATE.JL TOO +# DEFINITELY DUPLICATED CODE +############################################# - blas_lib = get_library_root(OpenBLAS32_jll, "JULIA_OPENBLAS_PATH") - if HAS_CUDA - cutensor_lib = get_library_root(CUTENSOR_jll, "JULIA_CUTENSOR_PATH") - end - - cupynumeric_path = cupynumeric_jll.artifact_dir +function check_jll(m::Module) + if !m.is_available() + m_host_cuda = cupynumeric_jll.host_platform["cuda"] - mode = load_preference(CNPreferences, "cunumeric_mode", CNPreferences.MODE_JLL) + if(m_host_cuda == "none") + error("$(string(m)) installed but not available on this platform.\n $(string(cupynumeric_jll.host_platform))") + end - # if developer mode - if mode == CNPreferences.MODE_JLL - cunumeric_wrapper_lib = joinpath(cunumeric_jl_wrapper_jll.artifact_dir, "lib") - elseif mode == CNPreferences.MODE_DEVELOPER - use_cupynumeric_jll = load_preference( - CNPreferences, "cunumeric_use_jll", true - ) - if use_cupynumeric_jll == false - cupynumeric_path = load_preference( - CNPreferences, "cunumeric_path", nothing - ) - check_cupynumeric_install(cupynumeric_path) + v_host_cuda = VersionNumber(m_host_cuda) + valid_cuda_version = Legate.MIN_CUDA_VERSION <= v_host_cuda <= Legate.MAX_CUDA_VERSION + if !valid_cuda_version + error("$(string(m)) installed but not available on this platform. Host CUDA ver: $(v_host_cuda) not in range supported by $(string(m)): $(MIN_CUDA_VERSION)-$(MAX_CUDA_VERSION).") + else + error("$(string(m)) installed but not available on this platform. Unknown reason.") end - cunumeric_wrapper_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build", "lib") - # if conda - elseif mode == CNPreferences.MODE_CONDA - @warn "mode = conda may break. We are using a subset of libraries from conda." - conda_env = load_preference(CNPreferences, "cunumeric_conda_env", nothing) - check_cupynumeric_install(conda_env) - cupynumeric_path = conda_env - cutensor_lib = joinpath(conda_env, "lib") end +end + +function find_paths( + mode::String; + cupynumeric_jll_module::Union{Module, Nothing} = nothing, + cupynumeric_jll_wrapper_module::Union{Module, Nothing} = nothing + ) + libcupynumeric_path, libcupynumeric_wrapper_path = + cuNumeric._find_paths(CNPreferences.to_mode(mode), cupynumeric_jll_module, cupynumeric_jll_wrapper_module) + set_preferences!(CNPreferences, "CUPYNUMERIC_LIBDIR" => libcupynumeric_path, force=true) + set_preferences!(CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR" => libcupynumeric_wrapper_path, force=true) +end - cupynumeric_lib = joinpath(cupynumeric_path, "lib") - if haskey(ENV, "JULIA_TBLIS_PATH") - tblis_lib = get(ENV, "JULIA_TBLIS_PATH", "0") + +function _find_paths( + mode::CNPreferences.JLL, + cupynumeric_jll_module::Module, + cupynumeric_jll_wrapper_module::Module + ) + check_jll(cupynumeric_jll_module) + check_jll(cupynumeric_jll_wrapper_module) + legate_lib_dir = joinpath(cupynumeric_jll_module.artifact_dir, "lib") + legate_wrapper_libdir = joinpath(cupynumeric_jll_wrapper_module.artifact_dir, "lib") + return legate_lib_dir, legate_wrapper_libdir +end + +function _find_paths( + mode::CNPreferences.Developer, + cupynumeric_jll_module::Module, + cupynumeric_jll_wrapper_module::Nothing + ) + + cupynumeric_path = "" + use_cupynumeric_jll = load_preference(CNPreferences, "cupynumeric_use_jll", true) + + if use_cupynumeric_jll == false + cupynumeric_path = load_preference(CNPreferences, "cupynumeric_path", nothing) + check_cupynumeric_install(cupynumeric_path) else - tblis_lib = cupynumeric_lib # cupynumeric libpath will by default contain tblis - end + check_jll(cupynumeric_jll_module) + cupynumeric_path = cupynumeric_jll.artifact_dir + end - if HAS_CUDA - set_preferences!(CNPreferences, "CUTENSOR_LIB" => cutensor_lib; force=true) - end + pkg_root = abspath(joinpath(@__DIR__, "../")) + legate_wrapper_lib = joinpath(pkg_root, "lib", "cupynumeric_jl_wrapper", "build", "lib") - set_preferences!(CNPreferences, "BLAS_LIB" => blas_lib; force=true) - set_preferences!(CNPreferences, "TBLIS_LIB" => tblis_lib; force=true) - set_preferences!(CNPreferences, "CUPYNUMERIC_LIB" => cupynumeric_lib; force=true) - set_preferences!(CNPreferences, "CUNUMERIC_WRAPPER_LIB" => cunumeric_wrapper_lib; force=true) + return joinpath(legate_path, "lib"), legate_wrapper_lib end + +function _find_paths( + mode::CNPreferences.Conda, + cupynumeric_jll_module::Nothing, + cupynumeric_jll_wrapper_module::Module + ) + + @warn "mode = conda may break. We are using a subset of libraries from conda." + + conda_env = load_preference(CNPreferences, "legate_conda_env", nothing) + isnothing(conda_env) && error("legate_conda_env preference must be set in LocalPreferences.toml when using conda mode") + + check_legate_install(conda_env) + legate_path = conda_env + check_jll(cupynumeric_jll_wrapper_module) + legate_wrapper_lib = joinpath(cupynumeric_jll_wrapper_module.artifact_dir, "lib") + + return joinpath(legate_path, "lib"), legate_wrapper_lib +end + +# MPI, NCCL etc are found by Legate.find_dependency_paths +const DEPS_MAP = Dict( + "CUTENSOR" => "libcutensor", + "BLAS" => "libopenblas", + "TBLIS" => "libtblis" +) +function find_dependency_paths(::Type{CNPreferences.JLL}) + results = Dict{String, String}() + + paths_to_search = copy(cupynumeric_jll.LIBPATH_list) + + for (name, lib) in DEPS_MAP + results[name] = dirname(Libdl.find_library(lib, paths_to_search)) + end + return results +end + +find_dependency_paths(::Type{CNPreferences.Developer}) = Dict{String, String}() +find_dependency_paths(::Type{CNPreferences.Conda}) = Dict{String, String}() diff --git a/src/utilities/version.jl b/src/utilities/version.jl index 9225f051..b3e2495e 100644 --- a/src/utilities/version.jl +++ b/src/utilities/version.jl @@ -39,7 +39,7 @@ function version_config_setup() name = get(project, "name", "unknown") version = get(project, "version", "unknown") uuid = get(project, "uuid", "unknown") - compiler = get_cxx_version(libpath) + compiler = get_cxx_version(CUPYNUMERIC_LIB_PATH) julia_ver = VERSION hostname = gethostname() @@ -47,23 +47,13 @@ function version_config_setup() liblegate = Legate.LEGATE_LIBDIR liblegatewrapper = Legate.LEGATE_WRAPPER_LIBDIR - if Legate.LegatePreferences.MODE == "jll" - other_dirs = Legate.find_dependency_paths(Legate.JLL()) - else - other_dirs = Dict( - "HDF5" => "unknown", - "MPI" => "unknown", - "NCCL" => "unknown", - "CUDA_DRIVER" => "unknown", - "CUDA_RUNTIME" => "unknown", - ) - end + other_dirs = Dict() - libblas = BLAS_LIB - libcutensor = CUTENSOR_LIB - libcupynumeric = CUPYNUMERIC_LIB - libtblis = TBLIS_LIB - libcunumericwrapper = CUNUMERIC_WRAPPER_LIB + cn_mode = CNPreferences.to_mode(CNPreferences.MODE) + legate_mode = LegatePreferences.to_mode(LegatePreferences.MODE) + dirs1 = cuNumeric.find_dependency_paths(typeof(cn_mode)) + dirs2 = Legate.find_dependency_paths(typeof(legate_mode)) + other_dirs = merge(dirs1, dirs2) str = """ ─────────────────────────────────────────────── @@ -77,22 +67,26 @@ function version_config_setup() Hostname: $hostname Julia Version: $julia_ver C++ Compiler: $compiler - CUDA Driver: $(other_dirs["CUDA_DRIVER"]) - CUDA Runtime: $(other_dirs["CUDA_RUNTIME"]) + CUDA Driver: $(get(other_dirs,"CUDA_DRIVER","unknown")) + CUDA Runtime: $(get(other_dirs,"CUDA_RUNTIME","unknown")) Library Paths: Legate: $liblegate - cuPyNumeric: $libcupynumeric - BLAS: $libblas - TBLIS: $libtblis - CUTENSOR: $libcutensor - NCCL: $(other_dirs["NCCL"]) - MPI: $(other_dirs["MPI"]) - HDF5: $(other_dirs["HDF5"]) + cuPyNumeric: $(CUPYNUMERIC_LIBDIR) + BLAS: $(get(other_dirs,"BLAS","unknown")) + TBLIS: $(get(other_dirs,"TBLIS","unknown")) + CUTENSOR: $(get(other_dirs,"CUTENSOR","unknown")) + NCCL: $(get(other_dirs,"NCCL","unknown")) + MPI: $(get(other_dirs,"MPI","unknown")) + HDF5: $(get(other_dirs,"HDF5","unknown")) Wrappers: - cuNumeric $libcunumericwrapper + cuNumeric $(CUPYNUMERIC_WRAPPER_LIBDIR) Legate $liblegatewrapper + + Modes: + cuNumeric: $(CNPreferences.MODE) + Legate: $(LegatePreferences.MODE) ─────────────────────────────────────────────── """ return str diff --git a/test/runtests.jl b/test/runtests.jl index 60b90c18..1d255796 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,34 +18,16 @@ =# using Test -using cuNumeric - using LinearAlgebra -using CUDA -import CUDA: i32 - using Random import Random: rand -include("tests/util.jl") -include("tests/axpy.jl") -include("tests/axpy_advanced.jl") -include("tests/elementwise.jl") -include("tests/slicing.jl") -include("tests/gemm.jl") -include("tests/unary_tests.jl") -include("tests/binary_tests.jl") -include("tests/scoping.jl") -include("tests/scoping-advanced.jl") - -const VERBOSE = false - +const VERBOSE = get(ENV, "VERBOSE", "1") != "0" const run_gpu_tests = get(ENV, "GPUTESTS", "1") != "0" const run_cuda_tests = run_gpu_tests && CUDA.functional() +@info "Run CUDA Tests: $(run_cuda_tests)" -if VERBOSE - cuNumeric.versioninfo() -end +VERBOSE && cuNumeric.versioninfo() if run_gpu_tests && VERBOSE println(CUDA.versioninfo()) @@ -55,9 +37,24 @@ if run_gpu_tests && !CUDA.functional() error( "You asked for CUDA tests, but they are disabled because no functional CUDA device was detected." ) +else + using CUDA + import CUDA: i32 end -@info "Run CUDA Tests: $(run_cuda_tests)" +using cuNumeric + + +include("tests/util.jl") +include("tests/axpy.jl") +include("tests/axpy_advanced.jl") +include("tests/elementwise.jl") +include("tests/slicing.jl") +include("tests/gemm.jl") +include("tests/unary_tests.jl") +include("tests/binary_tests.jl") +include("tests/scoping.jl") +include("tests/scoping-advanced.jl") @testset verbose = true "AXPY" begin N = 100 From 15ef65acfbe53633097fef1d24c47a6b51804c1f Mon Sep 17 00:00:00 2001 From: ejmeitz Date: Mon, 12 Jan 2026 16:03:09 -0600 Subject: [PATCH 02/17] forgot to add pkg ext to Project.toml --- Project.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Project.toml b/Project.toml index 67cd775f..ae832d29 100644 --- a/Project.toml +++ b/Project.toml @@ -23,6 +23,12 @@ cunumeric_jl_wrapper_jll = "49048992-29d2-5fd1-994f-9cecf112d624" cupynumeric_jll = "2862d674-414d-5b0b-a494-b21f8deca547" libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7" +[weakdeps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + +[ext] +CUDAExt = "CUDA" + [compat] CNPreferences = "0.1.2" CxxWrap = "0.17" From 80bb7b1fdb0b23837831a195ca17d08ec39be526 Mon Sep 17 00:00:00 2001 From: ejmeitz Date: Mon, 12 Jan 2026 16:06:23 -0600 Subject: [PATCH 03/17] rearrange some things in runtests.jl --- test/runtests.jl | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1d255796..98bcd5f0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -27,23 +27,20 @@ const run_gpu_tests = get(ENV, "GPUTESTS", "1") != "0" const run_cuda_tests = run_gpu_tests && CUDA.functional() @info "Run CUDA Tests: $(run_cuda_tests)" -VERBOSE && cuNumeric.versioninfo() - -if run_gpu_tests && VERBOSE - println(CUDA.versioninfo()) -end +if run_gpu_tests + using CUDA + import CUDA: i32 + VERBOSE && println(CUDA.versioninfo()) +end if run_gpu_tests && !CUDA.functional() error( "You asked for CUDA tests, but they are disabled because no functional CUDA device was detected." ) -else - using CUDA - import CUDA: i32 end using cuNumeric - +VERBOSE && cuNumeric.versioninfo() include("tests/util.jl") include("tests/axpy.jl") From 4db8e5cb2d3e847faac80f11f5054381f5f030c9 Mon Sep 17 00:00:00 2001 From: ejmeitz Date: Mon, 12 Jan 2026 16:08:52 -0600 Subject: [PATCH 04/17] oops --- test/runtests.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 98bcd5f0..85e470d0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,7 +24,6 @@ import Random: rand const VERBOSE = get(ENV, "VERBOSE", "1") != "0" const run_gpu_tests = get(ENV, "GPUTESTS", "1") != "0" -const run_cuda_tests = run_gpu_tests && CUDA.functional() @info "Run CUDA Tests: $(run_cuda_tests)" if run_gpu_tests @@ -392,7 +391,7 @@ end end end -if run_cuda_tests +if run_gpu_tests include("tests/cuda/vecadd.jl") @testset verbose = true "CUDA Tests" begin cuda_unaryop(rtol(Float32)) From f0ce55c3aa1d11e5570cdf8df474412e9a6d090e Mon Sep 17 00:00:00 2001 From: ejmeitz Date: Mon, 12 Jan 2026 16:09:23 -0600 Subject: [PATCH 05/17] oops again --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 85e470d0..5f8f933f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,7 +24,7 @@ import Random: rand const VERBOSE = get(ENV, "VERBOSE", "1") != "0" const run_gpu_tests = get(ENV, "GPUTESTS", "1") != "0" -@info "Run CUDA Tests: $(run_cuda_tests)" +@info "Run GPU Tests: $(run_gpu_tests)" if run_gpu_tests using CUDA From 104c9a12adb0d63d5b6183e50cebff1bd0e8a71f Mon Sep 17 00:00:00 2001 From: krasow Date: Mon, 12 Jan 2026 23:57:47 -0600 Subject: [PATCH 06/17] add OpenBLAS32_jll back for developer mode. adjust some path. However, I'm currently gettting the same undefined symbol for legate::detail::processor_id() as discussed in the meeting --- Project.toml | 8 ++-- src/utilities/preference.jl | 76 ++++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/Project.toml b/Project.toml index ae832d29..da8e6554 100644 --- a/Project.toml +++ b/Project.toml @@ -13,6 +13,7 @@ LegatePreferences = "8028f36a-2b64-49e9-aa04-2d0933fd2ed9" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -26,16 +27,17 @@ libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7" [weakdeps] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" -[ext] -CUDAExt = "CUDA" - [compat] CNPreferences = "0.1.2" CxxWrap = "0.17" Legate = "0.1.0" LegatePreferences = "0.1.5" MacroTools = "0.5.16" +OpenBLAS32_jll = "0.3.29" StatsBase = "0.34" cunumeric_jl_wrapper_jll = "25.10.1" cupynumeric_jll = "25.10.1" julia = "1.10" + +[ext] +CUDAExt = "CUDA" diff --git a/src/utilities/preference.jl b/src/utilities/preference.jl index 249a8517..212c0c48 100644 --- a/src/utilities/preference.jl +++ b/src/utilities/preference.jl @@ -60,14 +60,18 @@ function check_jll(m::Module) if !m.is_available() m_host_cuda = cupynumeric_jll.host_platform["cuda"] - if(m_host_cuda == "none") - error("$(string(m)) installed but not available on this platform.\n $(string(cupynumeric_jll.host_platform))") + if (m_host_cuda == "none") + error( + "$(string(m)) installed but not available on this platform.\n $(string(cupynumeric_jll.host_platform))", + ) end v_host_cuda = VersionNumber(m_host_cuda) valid_cuda_version = Legate.MIN_CUDA_VERSION <= v_host_cuda <= Legate.MAX_CUDA_VERSION if !valid_cuda_version - error("$(string(m)) installed but not available on this platform. Host CUDA ver: $(v_host_cuda) not in range supported by $(string(m)): $(MIN_CUDA_VERSION)-$(MAX_CUDA_VERSION).") + error( + "$(string(m)) installed but not available on this platform. Host CUDA ver: $(v_host_cuda) not in range supported by $(string(m)): $(MIN_CUDA_VERSION)-$(MAX_CUDA_VERSION).", + ) else error("$(string(m)) installed but not available on this platform. Unknown reason.") end @@ -75,22 +79,24 @@ function check_jll(m::Module) end function find_paths( - mode::String; - cupynumeric_jll_module::Union{Module, Nothing} = nothing, - cupynumeric_jll_wrapper_module::Union{Module, Nothing} = nothing + mode::String; + cupynumeric_jll_module::Union{Module,Nothing}=nothing, + cupynumeric_jll_wrapper_module::Union{Module,Nothing}=nothing, +) + libcupynumeric_path, libcupynumeric_wrapper_path = cuNumeric._find_paths( + CNPreferences.to_mode(mode), cupynumeric_jll_module, cupynumeric_jll_wrapper_module + ) + set_preferences!(CNPreferences, "CUPYNUMERIC_LIBDIR" => libcupynumeric_path; force=true) + set_preferences!( + CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR" => libcupynumeric_wrapper_path; force=true ) - libcupynumeric_path, libcupynumeric_wrapper_path = - cuNumeric._find_paths(CNPreferences.to_mode(mode), cupynumeric_jll_module, cupynumeric_jll_wrapper_module) - set_preferences!(CNPreferences, "CUPYNUMERIC_LIBDIR" => libcupynumeric_path, force=true) - set_preferences!(CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR" => libcupynumeric_wrapper_path, force=true) end - function _find_paths( - mode::CNPreferences.JLL, - cupynumeric_jll_module::Module, - cupynumeric_jll_wrapper_module::Module - ) + mode::CNPreferences.JLL, + cupynumeric_jll_module::Module, + cupynumeric_jll_wrapper_module::Module, +) check_jll(cupynumeric_jll_module) check_jll(cupynumeric_jll_wrapper_module) legate_lib_dir = joinpath(cupynumeric_jll_module.artifact_dir, "lib") @@ -99,38 +105,38 @@ function _find_paths( end function _find_paths( - mode::CNPreferences.Developer, - cupynumeric_jll_module::Module, - cupynumeric_jll_wrapper_module::Nothing - ) - + mode::CNPreferences.Developer, + cupynumeric_jll_module::Module, + cupynumeric_jll_wrapper_module::Nothing, +) cupynumeric_path = "" use_cupynumeric_jll = load_preference(CNPreferences, "cupynumeric_use_jll", true) - + if use_cupynumeric_jll == false cupynumeric_path = load_preference(CNPreferences, "cupynumeric_path", nothing) check_cupynumeric_install(cupynumeric_path) else check_jll(cupynumeric_jll_module) cupynumeric_path = cupynumeric_jll.artifact_dir - end + end - pkg_root = abspath(joinpath(@__DIR__, "../")) - legate_wrapper_lib = joinpath(pkg_root, "lib", "cupynumeric_jl_wrapper", "build", "lib") + pkg_root = abspath(joinpath(@__DIR__, "../", "../")) + wrapper_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build", "lib") - return joinpath(legate_path, "lib"), legate_wrapper_lib + return joinpath(cupynumeric_path, "lib"), wrapper_lib end function _find_paths( - mode::CNPreferences.Conda, - cupynumeric_jll_module::Nothing, - cupynumeric_jll_wrapper_module::Module - ) - + mode::CNPreferences.Conda, + cupynumeric_jll_module::Nothing, + cupynumeric_jll_wrapper_module::Module, +) @warn "mode = conda may break. We are using a subset of libraries from conda." conda_env = load_preference(CNPreferences, "legate_conda_env", nothing) - isnothing(conda_env) && error("legate_conda_env preference must be set in LocalPreferences.toml when using conda mode") + isnothing(conda_env) && error( + "legate_conda_env preference must be set in LocalPreferences.toml when using conda mode" + ) check_legate_install(conda_env) legate_path = conda_env @@ -144,10 +150,10 @@ end const DEPS_MAP = Dict( "CUTENSOR" => "libcutensor", "BLAS" => "libopenblas", - "TBLIS" => "libtblis" + "TBLIS" => "libtblis", ) function find_dependency_paths(::Type{CNPreferences.JLL}) - results = Dict{String, String}() + results = Dict{String,String}() paths_to_search = copy(cupynumeric_jll.LIBPATH_list) @@ -157,5 +163,5 @@ function find_dependency_paths(::Type{CNPreferences.JLL}) return results end -find_dependency_paths(::Type{CNPreferences.Developer}) = Dict{String, String}() -find_dependency_paths(::Type{CNPreferences.Conda}) = Dict{String, String}() +find_dependency_paths(::Type{CNPreferences.Developer}) = Dict{String,String}() +find_dependency_paths(::Type{CNPreferences.Conda}) = Dict{String,String}() From 2ae9ca43211898dcc0ebf5228c1c7f786c8259e9 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 00:20:47 -0600 Subject: [PATCH 07/17] add TODO in testing script --- .githash | 2 +- test/runtests.jl | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.githash b/.githash index 78440496..3a1bc8f2 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -5f235a255f7b040ede53f151d0652d6352e087e9 +104c9a12adb0d63d5b6183e50cebff1bd0e8a71f diff --git a/test/runtests.jl b/test/runtests.jl index 5f8f933f..2f7e3d1e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,11 +26,11 @@ const VERBOSE = get(ENV, "VERBOSE", "1") != "0" const run_gpu_tests = get(ENV, "GPUTESTS", "1") != "0" @info "Run GPU Tests: $(run_gpu_tests)" -if run_gpu_tests +if run_gpu_tests using CUDA import CUDA: i32 VERBOSE && println(CUDA.versioninfo()) -end +end if run_gpu_tests && !CUDA.functional() error( @@ -41,6 +41,12 @@ end using cuNumeric VERBOSE && cuNumeric.versioninfo() +# TODO +# After loading cuNumeric, we should verify that the Legate config has set a GPU device +# Right now, if you have a gpu device, but your LEGATE_CONFIG is cpu only, +# @cuda_task will not be defined and tests will fail confusingly. +# We should error out more gracefully in this situation. + include("tests/util.jl") include("tests/axpy.jl") include("tests/axpy_advanced.jl") From 6695cb1ec9ed1e81133034a6a9835292e4496f94 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 15:13:40 -0600 Subject: [PATCH 08/17] update docs.yml and build.jl --- Project.toml | 2 -- deps/build.jl | 10 +++++----- scripts/build_cpp_wrapper.sh | 10 ++++------ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Project.toml b/Project.toml index da8e6554..680faccf 100644 --- a/Project.toml +++ b/Project.toml @@ -13,7 +13,6 @@ LegatePreferences = "8028f36a-2b64-49e9-aa04-2d0933fd2ed9" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -33,7 +32,6 @@ CxxWrap = "0.17" Legate = "0.1.0" LegatePreferences = "0.1.5" MacroTools = "0.5.16" -OpenBLAS32_jll = "0.3.29" StatsBase = "0.34" cunumeric_jl_wrapper_jll = "25.10.1" cupynumeric_jll = "25.10.1" diff --git a/deps/build.jl b/deps/build.jl index 1196123d..a5d6894a 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -114,7 +114,7 @@ function build_jlcxxwrap(repo_root) end function build_cpp_wrapper( - repo_root, cupynumeric_loc, legate_loc, blas_lib, install_root + repo_root, cupynumeric_loc, legate_loc, install_root ) @info "libcunumeric_jl_wrapper: Building C++ Wrapper Library" if isdir(install_root) @@ -126,7 +126,7 @@ function build_cpp_wrapper( build_cpp_wrapper = joinpath(repo_root, "scripts/build_cpp_wrapper.sh") nthreads = Threads.nthreads() - bld_command = `$build_cpp_wrapper $repo_root $cupynumeric_loc $legate_loc $blas_lib $install_root $nthreads` + bld_command = `$build_cpp_wrapper $repo_root $cupynumeric_loc $legate_loc $install_root $nthreads` # write out a bash script for debugging cmd_str = join(bld_command.exec, " ") @@ -181,10 +181,10 @@ function build(mode) legate_lib = Legate.get_install_liblegate() cupynumeric_lib = load_preference(CNPreferences, "CUPYNUMERIC_LIB", nothing) - blas_lib = load_preference(CNPreferences, "BLAS_LIB", nothing) + # blas_lib = load_preference(CNPreferences, "BLAS_LIB", nothing) cupynumeric_lib = replace_nothing_conda_jll(mode, cupynumeric_lib, :cupynumeric_jll) - blas_lib = replace_nothing_jll(blas_lib, :OpenBLAS32_jll) + # blas_lib = replace_nothing_jll(blas_lib, :OpenBLAS32_jll) if mode == CNPreferences.MODE_DEVELOPER install_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build") @@ -197,7 +197,7 @@ function build(mode) ) end build_cpp_wrapper( - pkg_root, cupynumeric_root, up_dir(legate_lib), blas_lib, + pkg_root, cupynumeric_root, up_dir(legate_lib), install_lib, ) end diff --git a/scripts/build_cpp_wrapper.sh b/scripts/build_cpp_wrapper.sh index 13ad77f5..39a502e1 100755 --- a/scripts/build_cpp_wrapper.sh +++ b/scripts/build_cpp_wrapper.sh @@ -1,16 +1,15 @@ set -e # Check if exactly one argument is provided -if [[ $# -ne 6 ]]; then - echo "Usage: $0 " +if [[ $# -ne 5 ]]; then + echo "Usage: $0 " exit 1 fi CUNUMERICJL_ROOT_DIR=$1 # this is the repo root of cunumeric.jl CUPYNUMERIC_ROOT_DIR=$2 LEGATE_ROOT_DIR=$3 -BLAS_LIB_DIR=$4 -INSTALL_DIR=$5 -NTHREADS=$6 +INSTALL_DIR=$4 +NTHREADS=$5 # Check if the provided argument is a valid directory @@ -49,7 +48,6 @@ if [[ ! -f "$BUILD_DIR/CMakeCache.txt" ]]; then -D CMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ -D CMAKE_PREFIX_PATH="$CUPYNUMERIC_ROOT_DIR;$LEGATE_ROOT_DIR;" \ -D CUPYNUMERIC_PATH="$CUPYNUMERIC_ROOT_DIR" \ - -D BLAS_LIBRARIES="$BLAS_LIB_DIR/libopenblas.so" \ -D PROJECT_INSTALL_PATH="$INSTALL_DIR" \ -D CMAKE_BUILD_TYPE=Releases else From 12de09ae4434e4ed0b773421e9d1ad64723c7991 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 15:15:31 -0600 Subject: [PATCH 09/17] update 13.0.0 container for developer.yml --- .githash | 2 +- .github/workflows/developer.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.githash b/.githash index 3a1bc8f2..51402a8c 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -104c9a12adb0d63d5b6183e50cebff1bd0e8a71f +6695cb1ec9ed1e81133034a6a9835292e4496f94 diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 1857b47f..d6c3bdc6 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest container: # need nvcc - image: nvidia/cuda:12.4.0-devel-ubuntu22.04 + image: nvidia/cuda:13.0.0-devel-ubuntu22.04 # options: --gpus all defaults: run: From db62d8024830b3e8e8e005abd96e3fce2c4773a4 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 15:17:20 -0600 Subject: [PATCH 10/17] update docs.yml : --- .githash | 2 +- .github/workflows/docs.yml | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/.githash b/.githash index 51402a8c..d4d6dc31 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -6695cb1ec9ed1e81133034a6a9835292e4496f94 +12de09ae4434e4ed0b773421e9d1ad64723c7991 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e8f82d94..45540153 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -66,22 +66,6 @@ jobs: julia --color=yes -e 'using Pkg; Pkg.rm("cuNumeric")' || true # ensure clean env julia --color=yes -e 'using Pkg; Pkg.rm("Legate")' || true # ensure clean env - - name: Setup CUDA runtime version - run: | - julia --color=yes --project=docs -e ' - using Pkg - Pkg.add("CUDA") - using CUDA - CUDA.set_runtime_version!(v"12.4") - ' - - name: Set LD_LIBRARY_PATH for CUDA_Driver_jll - run: | - echo "LD_LIBRARY_PATH=$(julia -e 'using Pkg; \ - Pkg.add(name = "CUDA_Driver_jll", version = "0.12.1"); \ - using CUDA_Driver_jll; \ - print(joinpath(CUDA_Driver_jll.artifact_dir, "lib"))' \ - ):$LD_LIBRARY_PATH" >> $GITHUB_ENV - - name: Clone cuNumeric at ref run: | REF=${{ steps.pkg.outputs.ref }} @@ -104,7 +88,6 @@ jobs: run: | julia --color=yes --project=docs -e ' using Pkg - Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/Legate.jl", rev = "main", subdir = "lib/LegatePreferences")) Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/Legate.jl", rev = "main")) Pkg.develop(path = joinpath(homedir(), ".julia", "dev", "cuNumeric/lib/CNPreferences")) Pkg.develop(path = joinpath(homedir(), ".julia", "dev", "cuNumeric")) From 2c9d0088fbb269f9f9bc34c93229e72b2d2e507e Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 16:11:57 -0600 Subject: [PATCH 11/17] remove cuda depend on container that runs developer test --- .github/workflows/developer.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index d6c3bdc6..7fb18e0d 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -40,10 +40,6 @@ jobs: actions: write # runs-on: [self-hosted, linux, x64] runs-on: ubuntu-latest - container: - # need nvcc - image: nvidia/cuda:13.0.0-devel-ubuntu22.04 - # options: --gpus all defaults: run: shell: bash From 5113107e8dc0fa3dfc8799a830bd9491af33d711 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 16:17:00 -0600 Subject: [PATCH 12/17] update image id --- .github/workflows/developer.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 7fb18e0d..317775f0 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -40,6 +40,8 @@ jobs: actions: write # runs-on: [self-hosted, linux, x64] runs-on: ubuntu-latest + container: + image: ubuntu:22.04 defaults: run: shell: bash From da546f765fef66ce6ddf25f55e7a8dfba82ce4a9 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 16:39:13 -0600 Subject: [PATCH 13/17] GPUTESTS=0 --- .githash | 2 +- .github/workflows/developer.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.githash b/.githash index d4d6dc31..0b246ed5 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -12de09ae4434e4ed0b773421e9d1ad64723c7991 +5113107e8dc0fa3dfc8799a830bd9491af33d711 diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 317775f0..92ba83c4 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -123,4 +123,4 @@ jobs: - name: Perform Test run: | - julia --color=yes -e 'using Pkg; Pkg.test("cuNumeric")' + GPUTESTS=0 julia --color=yes -e 'using Pkg; Pkg.test("cuNumeric")' From 6d05e4373393ed2d3437161ef139ef7b99427629 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 17:53:25 -0600 Subject: [PATCH 14/17] clear cache? --- .githash | 2 +- .github/workflows/developer.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.githash b/.githash index 0b246ed5..a3332333 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -5113107e8dc0fa3dfc8799a830bd9491af33d711 +da546f765fef66ce6ddf25f55e7a8dfba82ce4a9 diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 92ba83c4..9c1a0300 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -96,7 +96,7 @@ jobs: id: julia-cache uses: julia-actions/cache@v2 with: - cache-name: julia-developer-ci-test-new + cache-name: julia-developer-ci - name: Cleanup old cuNumeric and Legate run: | julia --color=yes -e 'using Pkg; Pkg.rm("cuNumeric")' || true # ensure clean env From 3e3129a57f549a4be04e4cf72884f9e4b2d3aa1b Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 19:18:34 -0600 Subject: [PATCH 15/17] attempt to debug print logs --- .githash | 2 +- .github/workflows/developer.yml | 35 ++++++---------------------- deps/build.jl | 3 --- src/cuNumeric.jl | 41 +++++++++++++++++++++++---------- 4 files changed, 37 insertions(+), 44 deletions(-) diff --git a/.githash b/.githash index a3332333..b7d7649b 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -da546f765fef66ce6ddf25f55e7a8dfba82ce4a9 +6d05e4373393ed2d3437161ef139ef7b99427629 diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 9c1a0300..bf2a8074 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -52,37 +52,17 @@ jobs: uses: actions/checkout@v4 - name: bash run: source /etc/profile - - name: Get package spec - id: pkg - run: | - if [[ -n "${{ inputs.tag }}" ]]; then - echo "ref=${{ inputs.tag }}" >> $GITHUB_OUTPUT - echo "name=${{ inputs.tag }}" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_type }}" == "tag" ]]; then - echo "ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - elif [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "ref=${{ github.sha }}" >> $GITHUB_OUTPUT - echo "name=pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_type }}" == "branch" ]]; then - echo "ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - else - echo "ref=${{ github.sha }}" >> $GITHUB_OUTPUT - echo "name=dev" >> $GITHUB_OUTPUT - fi - VERSION=$(grep "^version = " Project.toml | cut -d'"' -f2) - echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Install basic utils + run: | + apt-get update && apt-get install -y wget curl git build-essential + curl -fsSL https://deb.nodesource.com/setup_20.x | bash - + apt-get install -y nodejs - name: Setup Julia uses: julia-actions/setup-julia@v2 with: version: '1.11' - - - name: Install basic utils - run: apt-get update && apt-get install -y wget curl git build-essential - - name: Install CMake 3.30.7 run: | @@ -111,13 +91,12 @@ jobs: using LegatePreferences; LegatePreferences.use_developer_mode(); Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/Legate.jl", rev = "main")) ' - julia --color=yes -e 'using Pkg; Pkg.build("Legate")' julia --color=yes -e ' using Pkg; - Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/cuNumeric.jl", rev = "${{ steps.pkg.outputs.ref }}", subdir = "lib/CNPreferences")) + Pkg.develop(PackageSpec(path = "lib/CNPreferences")) using CNPreferences; CNPreferences.use_developer_mode(); - Pkg.add(PackageSpec(url = "https://github.com/JuliaLegate/cuNumeric.jl", rev = "${{ steps.pkg.outputs.ref }}")) + Pkg.develop(PackageSpec(path = ".")) ' julia --color=yes -e 'using Pkg; Pkg.build("cuNumeric")' diff --git a/deps/build.jl b/deps/build.jl index a5d6894a..ebfa323a 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -181,10 +181,7 @@ function build(mode) legate_lib = Legate.get_install_liblegate() cupynumeric_lib = load_preference(CNPreferences, "CUPYNUMERIC_LIB", nothing) - # blas_lib = load_preference(CNPreferences, "BLAS_LIB", nothing) - cupynumeric_lib = replace_nothing_conda_jll(mode, cupynumeric_lib, :cupynumeric_jll) - # blas_lib = replace_nothing_jll(blas_lib, :OpenBLAS32_jll) if mode == CNPreferences.MODE_DEVELOPER install_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build") diff --git a/src/cuNumeric.jl b/src/cuNumeric.jl index 715a4376..b769eae2 100644 --- a/src/cuNumeric.jl +++ b/src/cuNumeric.jl @@ -48,7 +48,7 @@ include("utilities/preference.jl") find_paths( CNPreferences.MODE; cupynumeric_jll_module=cupynumeric_jll, - cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll + cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll, ) elseif CNPreferences.MODE == "developer" use_cupynumeric_jll = load_preference(CNPreferences, "legate_use_jll", true) @@ -57,7 +57,7 @@ elseif CNPreferences.MODE == "developer" find_paths( CNPreferences.MODE; cupynumeric_jll_module=cupynumeric_jll, - cupynumeric_jll_wrapper_module=nothing + cupynumeric_jll_wrapper_module=nothing, ) else find_paths(CNPreferences.MODE) @@ -67,27 +67,45 @@ elseif CNPreferences.MODE == "conda" find_paths( CNPreferences.MODE, cupynumeric_jll_module=nothing, - cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll + cupynumeric_jll_wrapper_module=cunumeric_jl_wrapper_jll, ) else - error("cuNumeric.jl: Unknown mode $(CNPreferences.MODE). Must be one of 'jll', 'developer', or 'conda'.") + error( + "cuNumeric.jl: Unknown mode $(CNPreferences.MODE). Must be one of 'jll', 'developer', or 'conda'.", + ) end - const CUPYNUMERIC_LIBDIR = load_preference(CNPreferences, "CUPYNUMERIC_LIBDIR", nothing) -const CUPYNUMERIC_WRAPPER_LIBDIR = load_preference(CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR", nothing) +const CUPYNUMERIC_WRAPPER_LIBDIR = load_preference( + CNPreferences, "CUPYNUMERIC_WRAPPER_LIBDIR", nothing +) const libnda = joinpath(CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_c_wrapper.so") -const CUPYNUMERIC_WRAPPER_LIB_PATH = joinpath(CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_jl_wrapper.so") +const CUPYNUMERIC_WRAPPER_LIB_PATH = joinpath( + CUPYNUMERIC_WRAPPER_LIBDIR, "libcunumeric_jl_wrapper.so" +) const CUPYNUMERIC_LIB_PATH = joinpath(CUPYNUMERIC_LIBDIR, "libcupynumeric.so") -(isnothing(CUPYNUMERIC_LIBDIR) || isnothing(CUPYNUMERIC_WRAPPER_LIBDIR)) && error("cuNumeric.jl: CUPYNUMERIC_LIBDIR or CUPYNUMERIC_WRAPPER_LIBDIR preference not set. Check LocalPreferences.toml") +(isnothing(CUPYNUMERIC_LIBDIR) || isnothing(CUPYNUMERIC_WRAPPER_LIBDIR)) && error( + "cuNumeric.jl: CUPYNUMERIC_LIBDIR or CUPYNUMERIC_WRAPPER_LIBDIR preference not set. Check LocalPreferences.toml", +) if !isfile(CUPYNUMERIC_WRAPPER_LIB_PATH) - error("Developer mode: You need to call Pkg.build()") + # Print build error logs if available + deps_dir = joinpath(dirname(@__DIR__), "../", "deps") + for errfile in ["cpp_wrapper.err", "libcxxwrap.err"] + errpath = joinpath(deps_dir, errfile) + if isfile(errpath) + println("\n=== Contents of $errfile ===") + println(read(errpath, String)) + println("=== End of $errfile ===\n") + end + end + error( + "Developer mode: You need to call Pkg.build(). Library $CUPYNUMERIC_WRAPPER_LIB_PATH not found.", + ) end - @wrapmodule(() -> CUPYNUMERIC_WRAPPER_LIB_PATH) # custom GC @@ -133,7 +151,6 @@ end global cuNumeric_config_str::String = "" - @doc""" versioninfo() @@ -147,7 +164,7 @@ end # Runtime initilization function __init__() CNPreferences.check_unchanged() - + Libdl.dlopen(CUPYNUMERIC_LIB_PATH, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) Libdl.dlopen(CUPYNUMERIC_WRAPPER_LIB_PATH, Libdl.RTLD_GLOBAL | Libdl.RTLD_NOW) From b1468726ba4827ab3e5ebb6e96ad94b9df78aa46 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 20:06:22 -0600 Subject: [PATCH 16/17] readd blas --- .githash | 2 +- Project.toml | 2 ++ deps/build.jl | 9 ++++++--- scripts/build_cpp_wrapper.sh | 10 ++++++---- src/cuNumeric.jl | 10 +++++----- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.githash b/.githash index b7d7649b..de648619 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -6d05e4373393ed2d3437161ef139ef7b99427629 +3e3129a57f549a4be04e4cf72884f9e4b2d3aa1b diff --git a/Project.toml b/Project.toml index 680faccf..72c7ef19 100644 --- a/Project.toml +++ b/Project.toml @@ -13,6 +13,7 @@ LegatePreferences = "8028f36a-2b64-49e9-aa04-2d0933fd2ed9" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -32,6 +33,7 @@ CxxWrap = "0.17" Legate = "0.1.0" LegatePreferences = "0.1.5" MacroTools = "0.5.16" +OpenBLAS32_jll = "0.3" StatsBase = "0.34" cunumeric_jl_wrapper_jll = "25.10.1" cupynumeric_jll = "25.10.1" diff --git a/deps/build.jl b/deps/build.jl index ebfa323a..431f4565 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -114,7 +114,7 @@ function build_jlcxxwrap(repo_root) end function build_cpp_wrapper( - repo_root, cupynumeric_loc, legate_loc, install_root + repo_root, cupynumeric_loc, legate_loc, blas_loc, install_root ) @info "libcunumeric_jl_wrapper: Building C++ Wrapper Library" if isdir(install_root) @@ -126,7 +126,7 @@ function build_cpp_wrapper( build_cpp_wrapper = joinpath(repo_root, "scripts/build_cpp_wrapper.sh") nthreads = Threads.nthreads() - bld_command = `$build_cpp_wrapper $repo_root $cupynumeric_loc $legate_loc $install_root $nthreads` + bld_command = `$build_cpp_wrapper $repo_root $cupynumeric_loc $legate_loc $blas_loc $install_root $nthreads` # write out a bash script for debugging cmd_str = join(bld_command.exec, " ") @@ -181,7 +181,10 @@ function build(mode) legate_lib = Legate.get_install_liblegate() cupynumeric_lib = load_preference(CNPreferences, "CUPYNUMERIC_LIB", nothing) + blas_lib = load_preference(CNPreferences, "BLAS_LIB", nothing) + cupynumeric_lib = replace_nothing_conda_jll(mode, cupynumeric_lib, :cupynumeric_jll) + blas_lib = replace_nothing_jll(blas_lib, :OpenBLAS32_jll) if mode == CNPreferences.MODE_DEVELOPER install_lib = joinpath(pkg_root, "lib", "cunumeric_jl_wrapper", "build") @@ -194,7 +197,7 @@ function build(mode) ) end build_cpp_wrapper( - pkg_root, cupynumeric_root, up_dir(legate_lib), + pkg_root, cupynumeric_root, up_dir(legate_lib), up_dir(blas_lib), install_lib, ) end diff --git a/scripts/build_cpp_wrapper.sh b/scripts/build_cpp_wrapper.sh index 39a502e1..e16c30ed 100755 --- a/scripts/build_cpp_wrapper.sh +++ b/scripts/build_cpp_wrapper.sh @@ -1,15 +1,16 @@ set -e # Check if exactly one argument is provided -if [[ $# -ne 5 ]]; then - echo "Usage: $0 " +if [[ $# -ne 6 ]]; then + echo "Usage: $0 " exit 1 fi CUNUMERICJL_ROOT_DIR=$1 # this is the repo root of cunumeric.jl CUPYNUMERIC_ROOT_DIR=$2 LEGATE_ROOT_DIR=$3 -INSTALL_DIR=$4 -NTHREADS=$5 +BLAS_ROOT_DIR=$4 +INSTALL_DIR=$5 +NTHREADS=$6 # Check if the provided argument is a valid directory @@ -48,6 +49,7 @@ if [[ ! -f "$BUILD_DIR/CMakeCache.txt" ]]; then -D CMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ -D CMAKE_PREFIX_PATH="$CUPYNUMERIC_ROOT_DIR;$LEGATE_ROOT_DIR;" \ -D CUPYNUMERIC_PATH="$CUPYNUMERIC_ROOT_DIR" \ + -D BLAS_LIBRARIES="$BLAS_LIB_DIR/libopenblas.so" \ -D PROJECT_INSTALL_PATH="$INSTALL_DIR" \ -D CMAKE_BUILD_TYPE=Releases else diff --git a/src/cuNumeric.jl b/src/cuNumeric.jl index b769eae2..77d9a9f3 100644 --- a/src/cuNumeric.jl +++ b/src/cuNumeric.jl @@ -71,7 +71,7 @@ elseif CNPreferences.MODE == "conda" ) else error( - "cuNumeric.jl: Unknown mode $(CNPreferences.MODE). Must be one of 'jll', 'developer', or 'conda'.", + "cuNumeric.jl: Unknown mode $(CNPreferences.MODE). Must be one of 'jll', 'developer', or 'conda'." ) end @@ -87,14 +87,14 @@ const CUPYNUMERIC_WRAPPER_LIB_PATH = joinpath( const CUPYNUMERIC_LIB_PATH = joinpath(CUPYNUMERIC_LIBDIR, "libcupynumeric.so") (isnothing(CUPYNUMERIC_LIBDIR) || isnothing(CUPYNUMERIC_WRAPPER_LIBDIR)) && error( - "cuNumeric.jl: CUPYNUMERIC_LIBDIR or CUPYNUMERIC_WRAPPER_LIBDIR preference not set. Check LocalPreferences.toml", + "cuNumeric.jl: CUPYNUMERIC_LIBDIR or CUPYNUMERIC_WRAPPER_LIBDIR preference not set. Check LocalPreferences.toml" ) if !isfile(CUPYNUMERIC_WRAPPER_LIB_PATH) # Print build error logs if available - deps_dir = joinpath(dirname(@__DIR__), "../", "deps") + deps = joinpath(dirname(@__DIR__), "deps") for errfile in ["cpp_wrapper.err", "libcxxwrap.err"] - errpath = joinpath(deps_dir, errfile) + errpath = joinpath(deps, errfile) if isfile(errpath) println("\n=== Contents of $errfile ===") println(read(errpath, String)) @@ -102,7 +102,7 @@ if !isfile(CUPYNUMERIC_WRAPPER_LIB_PATH) end end error( - "Developer mode: You need to call Pkg.build(). Library $CUPYNUMERIC_WRAPPER_LIB_PATH not found.", + "Developer mode: You need to call Pkg.build(). Library $CUPYNUMERIC_WRAPPER_LIB_PATH not found." ) end From 500755d2fcf51239491cd368848fa3f227122a77 Mon Sep 17 00:00:00 2001 From: krasow Date: Tue, 13 Jan 2026 20:36:21 -0600 Subject: [PATCH 17/17] cuda env --- .githash | 2 +- .github/workflows/developer.yml | 1 + scripts/build_cpp_wrapper.sh | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.githash b/.githash index de648619..a82e977a 100644 --- a/.githash +++ b/.githash @@ -1 +1 @@ -3e3129a57f549a4be04e4cf72884f9e4b2d3aa1b +b1468726ba4827ab3e5ebb6e96ad94b9df78aa46 diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index bf2a8074..d933f948 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -47,6 +47,7 @@ jobs: shell: bash env: LEGATE_AUTO_CONFIG: 0 + NO_CUDA: ON steps: - name: Check out the repo uses: actions/checkout@v4 diff --git a/scripts/build_cpp_wrapper.sh b/scripts/build_cpp_wrapper.sh index e16c30ed..6466d787 100755 --- a/scripts/build_cpp_wrapper.sh +++ b/scripts/build_cpp_wrapper.sh @@ -42,10 +42,14 @@ fi echo $LEGATE_ROOT_DIR +# Default to OFF (CUDA support enabled), but allow override via environment variable +NO_CUDA=${NO_CUDA:-OFF} + if [[ ! -f "$BUILD_DIR/CMakeCache.txt" ]]; then echo "Configuring project..." cmake -S "$CUNUMERIC_WRAPPER_SOURCE" -B "$BUILD_DIR" \ -D BINARYBUILDER=OFF \ + -D NOCUDA=$NO_CUDA \ -D CMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ -D CMAKE_PREFIX_PATH="$CUPYNUMERIC_ROOT_DIR;$LEGATE_ROOT_DIR;" \ -D CUPYNUMERIC_PATH="$CUPYNUMERIC_ROOT_DIR" \