From 954f23716ffd9234798aa88cf613f71a8d974899 Mon Sep 17 00:00:00 2001 From: krasow Date: Thu, 6 Aug 2026 21:38:13 -0500 Subject: [PATCH 1/6] fix 1.12 finalizer threading issues w/ queue read by main thread --- src/cuNumeric.jl | 4 ++- src/memory.jl | 49 ++++++++++++++++++++++++++++++++++- src/ndarray/detail/ndarray.jl | 29 +++++++++++++++++---- 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/cuNumeric.jl b/src/cuNumeric.jl index 1c55b373d..2a5bcf653 100644 --- a/src/cuNumeric.jl +++ b/src/cuNumeric.jl @@ -191,7 +191,7 @@ end getargv(a::ArgcArgv) = Base.unsafe_convert(CxxPtr{CxxPtr{CxxChar}}, a.argv) function my_on_exit() - # @info "Cleaning Up cuNumeric" + return drain_pending_frees!() # flush before Legate tears down end global cuNumeric_config_str::String = "" @@ -215,6 +215,8 @@ function _start_runtime() # AA = ArgcArgv([Base.julia_cmd()[1]]) cuNumeric.initialize_cunumeric(AA.argc, getargv(AA)) + _init_deferred_free!() # record launch thread for deferred frees (memory.jl) + # setup /src/memory.jl cuNumeric.init_gc!() diff --git a/src/memory.jl b/src/memory.jl index b71866288..02deb81f6 100644 --- a/src/memory.jl +++ b/src/memory.jl @@ -1,4 +1,50 @@ -using Base.Threads: Atomic, atomic_add!, atomic_sub!, atomic_xchg! +using Base.Threads: Atomic, atomic_add!, atomic_sub!, atomic_xchg!, SpinLock + +# Legate only permits handle destruction on the launch thread, but GC finalizers can +# run on another thread (e.g. 1.12's interactive thread), so they enqueue here and the +# launch thread drains later. Presized buffers keep the finalizer enqueue alloc-free. +const _RUNTIME_TID = Ref{Int}(0) +const _free_lock = SpinLock() +const _free_queue = Ptr{Cvoid}[] # producers (finalizers) push +const _free_drain = Ptr{Cvoid}[] # launch thread swaps into here to free + +function _init_deferred_free!() + _RUNTIME_TID[] = Threads.threadid() + sizehint!(_free_queue, 1 << 16) + sizehint!(_free_drain, 1 << 16) + return nothing +end + +# Runs in finalizers on any thread: no Legate call, no block, no steady-state alloc. +@inline function _enqueue_free!(ptr::Ptr{Cvoid}) + ptr == C_NULL && return nothing + lock(_free_lock) + push!(_free_queue, ptr) + unlock(_free_lock) + return nothing +end + +@doc""" + drain_pending_frees!() + +Destroy NDArray handles queued by finalizers. No-op off the launch thread. Called +automatically from the op/allocation path, so user code rarely needs it. +""" +function drain_pending_frees!() + Threads.threadid() == _RUNTIME_TID[] || return nothing + isempty(_free_queue) && return nothing + + lock(_free_lock) + append!(_free_drain, _free_queue) + empty!(_free_queue) + unlock(_free_lock) + + for ptr in _free_drain + nda_destroy_array(ptr) + end + empty!(_free_drain) + return nothing +end query_total_device_memory() = ccall((:nda_query_total_device_memory, libnda), Int64, ()) @@ -118,6 +164,7 @@ end function _collect!(full::Bool) GC.gc(full) + drain_pending_frees!() # free what GC just enqueued recalibrate_allocator!() atomic_xchg!(post_gc_device_bytes, current_device_bytes[]) atomic_xchg!(post_gc_host_bytes, current_host_bytes[]) diff --git a/src/ndarray/detail/ndarray.jl b/src/ndarray/detail/ndarray.jl index 102d9dc43..5d4d621c0 100644 --- a/src/ndarray/detail/ndarray.jl +++ b/src/ndarray/detail/ndarray.jl @@ -8,9 +8,15 @@ struct Slice stop::Int64 end +# All op/allocation submissions pass through here; flush queued frees first so they +# land in program order relative to operations rather than at arbitrary GC points. macro task_scope(scope_name, body) - TASK_SCOPE_NAMES || return esc(body) + TASK_SCOPE_NAMES || return quote + drain_pending_frees!() + $(esc(body)) + end return quote + drain_pending_frees!() Legate.with_scope($(esc(scope_name))) do return $(esc(body)) end @@ -52,7 +58,7 @@ mutable struct NDArray{T,N,PADDED,P} <: AbstractNDArray{T,N} nbytes = cuNumeric.nda_nbytes(ptr) cuNumeric.register_alloc!(nbytes) handle = new{T,N,false,Nothing}(ptr, nbytes, nothing, nothing) - finalizer(destroy!, handle) + finalizer(_finalize_ndarray!, handle) return handle end @@ -61,11 +67,24 @@ mutable struct NDArray{T,N,PADDED,P} <: AbstractNDArray{T,N} nbytes = cuNumeric.nda_nbytes(ptr) cuNumeric.register_alloc!(nbytes) handle = new{T,N,false,P}(ptr, nbytes, nothing, parent) - finalizer(destroy!, handle) + finalizer(_finalize_ndarray!, handle) return handle end end +# May run off the launch thread, so defer the Legate free to drain_pending_frees!. +# Accounting is atomic and safe to do here immediately. +function _finalize_ndarray!(arr::NDArray) + ptr = arr.ptr + ptr == C_NULL && return nothing + arr.ptr = Ptr{Cvoid}(0) + nbytes = arr.nbytes + arr.nbytes = 0 + nbytes > 0 && register_free!(nbytes) + _enqueue_free!(ptr) + return nothing +end + @inline _is_ndarray_slice(arr::NDArray) = arr.parent isa NDArray """ @@ -562,7 +581,7 @@ function compare( end for CI in CartesianIndices(julia_array) - x = julia_array[CI]; + x = julia_array[CI] y = arr[Tuple(CI)...] if !isapprox(x, y; atol=atol, rtol=rtol) return false @@ -587,7 +606,7 @@ function compare(arr::NDArray{T,N}, arr2::NDArray{T,N}, atol::Real, rtol::Real) dims = shape(arr) for CI in CartesianIndices(dims) - x = arr[Tuple(CI)...]; + x = arr[Tuple(CI)...] y = arr2[Tuple(CI)...] if !isapprox(x, y; atol=atol, rtol=rtol) return false From 0bef0db5dc266b56ffc271f472f9a4498bc09629 Mon Sep 17 00:00:00 2001 From: krasow Date: Thu, 6 Aug 2026 22:58:19 -0500 Subject: [PATCH 2/6] developer builds run on all versions we actively support --- .buildkite/developer.pipeline.yml | 15 ++++++++++----- .github/workflows/developer.yml | 14 +++++++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.buildkite/developer.pipeline.yml b/.buildkite/developer.pipeline.yml index a3ed1fdf3..93db969d2 100644 --- a/.buildkite/developer.pipeline.yml +++ b/.buildkite/developer.pipeline.yml @@ -2,13 +2,14 @@ steps: - group: ":hammer: Developer" key: "developer" steps: - - label: "GPU - Julia 1.11 (source wrapper, fusion {{matrix.fusion}})" + - label: "GPU - Julia {{matrix.julia}} (source wrapper, fusion {{matrix.fusion}})" plugins: - JuliaCI/julia#v1: - version: "1.11" - # Developer builds install local wrapper overrides into the depot. - # Keep them in a separate cache so they cannot leak into JLL tests. - cache_dir: "${HOME}/.cache/julia-buildkite-plugin-developer" + version: "{{matrix.julia}}" + # Developer builds install version-specific (CxxWrap ABI) wrapper + # overrides into the depot, so key the cache per Julia version and keep + # it separate from the JLL cache. + cache_dir: "${HOME}/.cache/julia-buildkite-plugin-developer-{{matrix.julia}}" command: ".buildkite/run_developer_ci.sh" artifact_paths: - "deps/build.log" @@ -28,6 +29,10 @@ steps: CUNUMERIC_FUSION: "{{matrix.fusion}}" matrix: setup: + julia: + - "1.10" + - "1.11" + - "1.12" fusion: - "on" - "off" diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index ebeafdd1f..7654edec6 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -42,7 +42,7 @@ jobs: uses: ./.github/workflows/pkg_resolve.yml docs: - name: Developer CI test + name: Developer CI test - Julia ${{ matrix.julia }} needs: pkg_resolve permissions: contents: read @@ -50,6 +50,13 @@ jobs: attestations: write id-token: write actions: write + strategy: + fail-fast: false + matrix: + julia: + - '1.10' + - '1.11' + - '1.12' # runs-on: [self-hosted, linux, x64] runs-on: ubuntu-latest container: @@ -75,7 +82,7 @@ jobs: - name: Setup Julia uses: julia-actions/setup-julia@v2 with: - version: '1.11' + version: ${{ matrix.julia }} - name: Install CMake 3.30.7 run: | @@ -89,7 +96,8 @@ jobs: id: julia-cache uses: julia-actions/cache@v2 with: - cache-name: julia-developer-ci + # per-version: developer wrapper overrides are CxxWrap-ABI-specific + cache-name: julia-developer-ci-${{ matrix.julia }} # Parse commit message or PR body for [legate-branch: ]. # Example: include "[legate-branch: my-feature]" anywhere in the message. From cf1fadf53b6925cba83bab491f2c517cf5aaf2f4 Mon Sep 17 00:00:00 2001 From: krasow Date: Fri, 7 Aug 2026 00:05:36 -0500 Subject: [PATCH 3/6] try ENV skip --- deps/build.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deps/build.jl b/deps/build.jl index 0e0553adb..3b4775c12 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -19,7 +19,12 @@ using Pkg using Preferences + +# The build only needs Legate's paths/tooling, not a running runtime. +# Setting this env var prevents a segfault on Julia 1.12 +ENV["LEGATE_SKIP_RUNTIME"] = "true" using Legate + using CNPreferences using CUDACore: CUDACore From 9061c3c52d0bec84be11fb388c481a4505c85815 Mon Sep 17 00:00:00 2001 From: David Krasowska Date: Fri, 7 Aug 2026 00:16:36 -0500 Subject: [PATCH 4/6] retrigger CI --- deps/build.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/build.jl b/deps/build.jl index 3b4775c12..68ac071ce 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -21,7 +21,7 @@ using Pkg using Preferences # The build only needs Legate's paths/tooling, not a running runtime. -# Setting this env var prevents a segfault on Julia 1.12 +# Setting this env prevents a segfault on Julia 1.12 ENV["LEGATE_SKIP_RUNTIME"] = "true" using Legate From 4028c226e2fa23117d8c6a72cda41c890fd41675 Mon Sep 17 00:00:00 2001 From: krasow Date: Fri, 7 Aug 2026 01:03:02 -0500 Subject: [PATCH 5/6] caching is hard --- .buildkite/run_developer_ci.sh | 3 ++ .github/workflows/developer.yml | 49 +++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.buildkite/run_developer_ci.sh b/.buildkite/run_developer_ci.sh index 1a91c0b15..ca351ba9d 100755 --- a/.buildkite/run_developer_ci.sh +++ b/.buildkite/run_developer_ci.sh @@ -22,6 +22,9 @@ sh "$CMAKE_INSTALLER" --skip-license --prefix="$CMAKE_ROOT" export PATH="$CMAKE_ROOT/bin:$PATH" cmake --version +# Resolve against source checkouts / Legate override, not the pinned manifest. +rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml + LEGATE_BRANCH_INPUT="${BUILDKITE_MESSAGE:-}" if [[ "${BUILDKITE_PULL_REQUEST:-false}" =~ ^[0-9]+$ ]]; then echo "Reading Legate branch override from pull request #$BUILDKITE_PULL_REQUEST" diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 7654edec6..8be397fea 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -115,35 +115,42 @@ jobs: echo "No Legate.jl branch override — using JLL" fi - - name: Clone and develop Legate.jl branch override + - name: Clone Legate.jl branch override if: steps.legate-branch.outputs.branch != '' run: | git clone --depth 1 --branch ${{ steps.legate-branch.outputs.branch }} \ ${{ github.server_url }}/${{ github.repository_owner }}/Legate.jl.git \ /tmp/Legate.jl - julia --color=yes -e ' - using Pkg; - Pkg.develop(PackageSpec(path = "/tmp/Legate.jl/lib/LegatePreferences")) - Pkg.develop(PackageSpec(path = "/tmp/Legate.jl")) - using LegatePreferences; LegatePreferences.use_developer_mode(); - Pkg.build("Legate") - ' - - name: Setup cuNumeric.jl with build from src wrappers + # Fresh throwaway project (not the cached global env) so dev-mode prefs/paths + # don't get cached stale across runs; test/Project.toml is left untouched. + - name: Build and test cuNumeric (developer mode) + env: + GPUTESTS: "0" run: | - julia --color=yes -e ' - using Pkg; + CI_PROJECT="$(mktemp -d)" + + # Resolve against source checkouts / Legate override, not the pinned manifest. + rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml + + if [[ -n "${{ steps.legate-branch.outputs.branch }}" ]]; then + ( cd /tmp/Legate.jl && julia --color=yes --project="$CI_PROJECT" -e ' + using Pkg + Pkg.develop(PackageSpec(path = "lib/LegatePreferences")) + Pkg.develop(PackageSpec(path = ".")) + using LegatePreferences; LegatePreferences.use_developer_mode() + Pkg.build("Legate") + ' ) + fi + + julia --color=yes --project="$CI_PROJECT" -e ' + using Pkg Pkg.develop(PackageSpec(path = "lib/CNPreferences")) - using CNPreferences; CNPreferences.use_developer_mode(); + using CNPreferences; CNPreferences.use_developer_mode() Pkg.develop(PackageSpec(path = ".")) + Pkg.build("cuNumeric") ' - julia --color=yes --project=test -e ' - using Pkg; - Pkg.develop(PackageSpec(path = "lib/CNPreferences")) - using CNPreferences; CNPreferences.use_developer_mode(); - ' - julia --color=yes -e 'using Pkg; Pkg.build("cuNumeric")' - - name: Perform Test - run: | - GPUTESTS=0 julia --color=yes -e 'using Pkg; Pkg.test("cuNumeric")' + cp "$CI_PROJECT/LocalPreferences.toml" test/LocalPreferences.toml + + julia --color=yes --project="$CI_PROJECT" -e 'using Pkg; Pkg.test("cuNumeric")' From efc65360878fecec582f3f884be4c7b41eac73c1 Mon Sep 17 00:00:00 2001 From: krasow Date: Fri, 7 Aug 2026 01:49:29 -0500 Subject: [PATCH 6/6] trying pt 2 --- .buildkite/run_developer_ci.sh | 36 +++++++++++++++------------------ .github/workflows/developer.yml | 27 +++++++++++-------------- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/.buildkite/run_developer_ci.sh b/.buildkite/run_developer_ci.sh index ca351ba9d..ef4718772 100755 --- a/.buildkite/run_developer_ci.sh +++ b/.buildkite/run_developer_ci.sh @@ -2,8 +2,6 @@ set -euo pipefail -CI_PROJECT="$(mktemp -d)" - case "${CUNUMERIC_FUSION:-}" in on | off) ;; *) @@ -22,8 +20,9 @@ sh "$CMAKE_INSTALLER" --skip-license --prefix="$CMAKE_ROOT" export PATH="$CMAKE_ROOT/bin:$PATH" cmake --version -# Resolve against source checkouts / Legate override, not the pinned manifest. -rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml +# Clean slate so cached state doesn't leak across Julia versions. +rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml \ + LocalPreferences.toml test/LocalPreferences.toml LEGATE_BRANCH_INPUT="${BUILDKITE_MESSAGE:-}" if [[ "${BUILDKITE_PULL_REQUEST:-false}" =~ ^[0-9]+$ ]]; then @@ -44,41 +43,38 @@ if [[ "$LEGATE_BRANCH_INPUT" =~ legate[-_]branch:[[:space:]]*([A-Za-z0-9._/-]+) fi shopt -u nocasematch +# Develop into the workspace root so its members (incl. dev/) resolve to the override. if [[ -n "$LEGATE_BRANCH" ]]; then - LEGATE_CHECKOUT="$(mktemp -d)/Legate.jl" + export LEGATE_CHECKOUT="$(mktemp -d)/Legate.jl" echo "Using Legate.jl branch override: $LEGATE_BRANCH" git clone --depth 1 --branch "$LEGATE_BRANCH" \ https://github.com/JuliaLegate/Legate.jl.git "$LEGATE_CHECKOUT" git -C "$LEGATE_CHECKOUT" log -1 --format="Legate checkout: %D (%H)" - ( - cd "$LEGATE_CHECKOUT" - julia --color=yes --project="$CI_PROJECT" -e ' - using Pkg - Pkg.develop(PackageSpec(path = "lib/LegatePreferences")) - Pkg.develop(PackageSpec(path = ".")) - using LegatePreferences - LegatePreferences.use_developer_mode() - Pkg.build("Legate") - ' - ) + julia --color=yes --project=. -e ' + using Pkg + Pkg.develop(PackageSpec(path = joinpath(ENV["LEGATE_CHECKOUT"], "lib/LegatePreferences"))) + Pkg.develop(PackageSpec(path = ENV["LEGATE_CHECKOUT"])) + using LegatePreferences + LegatePreferences.use_developer_mode() + Pkg.build("Legate") + ' else echo "No Legate.jl branch override - using JLL" fi -julia --color=yes --project="$CI_PROJECT" -e ' +julia --color=yes --project=. -e ' using Pkg Pkg.develop(PackageSpec(path = "lib/CNPreferences")) using CNPreferences CNPreferences.use_developer_mode() CNPreferences.set_broadcast_fusion!(ENV["CUNUMERIC_FUSION"] == "on") CNPreferences.set_broadcast_fusion_min_ops!(1) - Pkg.develop(PackageSpec(path = ".")) Pkg.build("cuNumeric") ' -cp "$CI_PROJECT/LocalPreferences.toml" test/LocalPreferences.toml +cp LocalPreferences.toml test/LocalPreferences.toml -julia --color=yes --project="$CI_PROJECT" -e ' +julia --color=yes --project=. -e ' using Pkg Pkg.test("cuNumeric"; test_args = ["--quickfail"]) ' diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 8be397fea..dbca6de22 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -96,7 +96,7 @@ jobs: id: julia-cache uses: julia-actions/cache@v2 with: - # per-version: developer wrapper overrides are CxxWrap-ABI-specific + # per-version: dev wrappers are ABI-specific cache-name: julia-developer-ci-${{ matrix.julia }} # Parse commit message or PR body for [legate-branch: ]. @@ -122,35 +122,32 @@ jobs: ${{ github.server_url }}/${{ github.repository_owner }}/Legate.jl.git \ /tmp/Legate.jl - # Fresh throwaway project (not the cached global env) so dev-mode prefs/paths - # don't get cached stale across runs; test/Project.toml is left untouched. + # Develop into the workspace root so its members (incl. dev/) resolve to the override. - name: Build and test cuNumeric (developer mode) env: GPUTESTS: "0" + LEGATE_CHECKOUT: /tmp/Legate.jl run: | - CI_PROJECT="$(mktemp -d)" - - # Resolve against source checkouts / Legate override, not the pinned manifest. - rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml + rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml \ + LocalPreferences.toml test/LocalPreferences.toml if [[ -n "${{ steps.legate-branch.outputs.branch }}" ]]; then - ( cd /tmp/Legate.jl && julia --color=yes --project="$CI_PROJECT" -e ' + julia --color=yes --project=. -e ' using Pkg - Pkg.develop(PackageSpec(path = "lib/LegatePreferences")) - Pkg.develop(PackageSpec(path = ".")) + Pkg.develop(PackageSpec(path = joinpath(ENV["LEGATE_CHECKOUT"], "lib/LegatePreferences"))) + Pkg.develop(PackageSpec(path = ENV["LEGATE_CHECKOUT"])) using LegatePreferences; LegatePreferences.use_developer_mode() Pkg.build("Legate") - ' ) + ' fi - julia --color=yes --project="$CI_PROJECT" -e ' + julia --color=yes --project=. -e ' using Pkg Pkg.develop(PackageSpec(path = "lib/CNPreferences")) using CNPreferences; CNPreferences.use_developer_mode() - Pkg.develop(PackageSpec(path = ".")) Pkg.build("cuNumeric") ' - cp "$CI_PROJECT/LocalPreferences.toml" test/LocalPreferences.toml + cp LocalPreferences.toml test/LocalPreferences.toml - julia --color=yes --project="$CI_PROJECT" -e 'using Pkg; Pkg.test("cuNumeric")' + julia --color=yes --project=. -e 'using Pkg; Pkg.test("cuNumeric")'