fix 1.12 finalizer threading issues w/ queue read by main thread and update CI

.buildkite/developer.pipeline.yml

@@ -2,13 +2,14 @@ steps:
   - group: ":hammer: Developer"
     key: "developer"
     steps:
-      - label: "GPU - Julia 1.11 (source wrapper, fusion {{matrix.fusion}})"
+      - label: "GPU - Julia {{matrix.julia}} (source wrapper, fusion {{matrix.fusion}})"
         plugins:
           - JuliaCI/julia#v1:
-              version: "1.11"
-              # Developer builds install local wrapper overrides into the depot.
-              # Keep them in a separate cache so they cannot leak into JLL tests.
-              cache_dir: "${HOME}/.cache/julia-buildkite-plugin-developer"
+              version: "{{matrix.julia}}"
+              # Developer builds install version-specific (CxxWrap ABI) wrapper
+              # overrides into the depot, so key the cache per Julia version and keep
+              # it separate from the JLL cache.
+              cache_dir: "${HOME}/.cache/julia-buildkite-plugin-developer-{{matrix.julia}}"
         command: ".buildkite/run_developer_ci.sh"
         artifact_paths:
           - "deps/build.log"
@@ -28,6 +29,10 @@ steps:
           CUNUMERIC_FUSION: "{{matrix.fusion}}"
         matrix:
           setup:
+            julia:
+              - "1.10"
+              - "1.11"
+              - "1.12"
             fusion:
               - "on"
               - "off"

.buildkite/run_developer_ci.sh

@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-CI_PROJECT="$(mktemp -d)"
-
 case "${CUNUMERIC_FUSION:-}" in
     on | off) ;;
     *)
@@ -22,6 +20,10 @@ sh "$CMAKE_INSTALLER" --skip-license --prefix="$CMAKE_ROOT"
 export PATH="$CMAKE_ROOT/bin:$PATH"
 cmake --version
 
+# Clean slate so cached state doesn't leak across Julia versions.
+rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml \
+      LocalPreferences.toml test/LocalPreferences.toml
+
 LEGATE_BRANCH_INPUT="${BUILDKITE_MESSAGE:-}"
 if [[ "${BUILDKITE_PULL_REQUEST:-false}" =~ ^[0-9]+$ ]]; then
     echo "Reading Legate branch override from pull request #$BUILDKITE_PULL_REQUEST"
@@ -41,41 +43,38 @@ if [[ "$LEGATE_BRANCH_INPUT" =~ legate[-_]branch:[[:space:]]*([A-Za-z0-9._/-]+)
 fi
 shopt -u nocasematch
 
+# Develop into the workspace root so its members (incl. dev/) resolve to the override.
 if [[ -n "$LEGATE_BRANCH" ]]; then
-    LEGATE_CHECKOUT="$(mktemp -d)/Legate.jl"
+    export LEGATE_CHECKOUT="$(mktemp -d)/Legate.jl"
     echo "Using Legate.jl branch override: $LEGATE_BRANCH"
     git clone --depth 1 --branch "$LEGATE_BRANCH" \
         https://github.com/JuliaLegate/Legate.jl.git "$LEGATE_CHECKOUT"
     git -C "$LEGATE_CHECKOUT" log -1 --format="Legate checkout: %D (%H)"
-    (
-        cd "$LEGATE_CHECKOUT"
-        julia --color=yes --project="$CI_PROJECT" -e '
-            using Pkg
-            Pkg.develop(PackageSpec(path = "lib/LegatePreferences"))
-            Pkg.develop(PackageSpec(path = "."))
-            using LegatePreferences
-            LegatePreferences.use_developer_mode()
-            Pkg.build("Legate")
-        '
-    )
+    julia --color=yes --project=. -e '
+        using Pkg
+        Pkg.develop(PackageSpec(path = joinpath(ENV["LEGATE_CHECKOUT"], "lib/LegatePreferences")))
+        Pkg.develop(PackageSpec(path = ENV["LEGATE_CHECKOUT"]))
+        using LegatePreferences
+        LegatePreferences.use_developer_mode()
+        Pkg.build("Legate")
+    '
 else
     echo "No Legate.jl branch override - using JLL"
 fi
 
-julia --color=yes --project="$CI_PROJECT" -e '
+julia --color=yes --project=. -e '
     using Pkg
     Pkg.develop(PackageSpec(path = "lib/CNPreferences"))
     using CNPreferences
     CNPreferences.use_developer_mode()
     CNPreferences.set_broadcast_fusion!(ENV["CUNUMERIC_FUSION"] == "on")
     CNPreferences.set_broadcast_fusion_min_ops!(1)
-    Pkg.develop(PackageSpec(path = "."))
     Pkg.build("cuNumeric")
 '
 
-cp "$CI_PROJECT/LocalPreferences.toml" test/LocalPreferences.toml
+cp LocalPreferences.toml test/LocalPreferences.toml
 
-julia --color=yes --project="$CI_PROJECT" -e '
+julia --color=yes --project=. -e '
     using Pkg
     Pkg.test("cuNumeric"; test_args = ["--quickfail"])
 '

.github/workflows/developer.yml

@@ -42,14 +42,21 @@ jobs:
     uses: ./.github/workflows/pkg_resolve.yml
 
   docs:
-    name: Developer CI test
+    name: Developer CI test - Julia ${{ matrix.julia }}
     needs: pkg_resolve
     permissions:
       contents: read
       packages: write
       attestations: write
       id-token: write
       actions: write
+    strategy:
+      fail-fast: false
+      matrix:
+        julia:
+          - '1.10'
+          - '1.11'
+          - '1.12'
     # runs-on: [self-hosted, linux, x64]
     runs-on: ubuntu-latest
     container:
@@ -75,7 +82,7 @@ jobs:
       - name: Setup Julia
         uses: julia-actions/setup-julia@v2
         with:
-          version: '1.11'
+          version: ${{ matrix.julia }}
 
       - name: Install CMake 3.30.7
         run: |
@@ -89,7 +96,8 @@ jobs:
         id: julia-cache
         uses: julia-actions/cache@v2
         with:
-            cache-name: julia-developer-ci
+            # per-version: dev wrappers are ABI-specific
+            cache-name: julia-developer-ci-${{ matrix.julia }}
 
       # Parse commit message or PR body for [legate-branch: <branch>].
       # Example: include "[legate-branch: my-feature]" anywhere in the message.
@@ -107,35 +115,39 @@ jobs:
             echo "No Legate.jl branch override — using JLL"
           fi
 
-      - name: Clone and develop Legate.jl branch override
+      - name: Clone Legate.jl branch override
         if: steps.legate-branch.outputs.branch != ''
         run: |
           git clone --depth 1 --branch ${{ steps.legate-branch.outputs.branch }} \
             ${{ github.server_url }}/${{ github.repository_owner }}/Legate.jl.git \
             /tmp/Legate.jl
-          julia --color=yes -e '
-              using Pkg;
-              Pkg.develop(PackageSpec(path = "/tmp/Legate.jl/lib/LegatePreferences"))
-              Pkg.develop(PackageSpec(path = "/tmp/Legate.jl"))
-              using LegatePreferences; LegatePreferences.use_developer_mode();
-              Pkg.build("Legate")
-          '
 
-      - name: Setup cuNumeric.jl with build from src wrappers
+      # Develop into the workspace root so its members (incl. dev/) resolve to the override.
+      - name: Build and test cuNumeric (developer mode)
+        env:
+          GPUTESTS: "0"
+          LEGATE_CHECKOUT: /tmp/Legate.jl
         run: |
-          julia --color=yes -e '
-              using Pkg;
-              Pkg.develop(PackageSpec(path = "lib/CNPreferences"))
-              using CNPreferences; CNPreferences.use_developer_mode();
-              Pkg.develop(PackageSpec(path = "."))
-          '
-          julia --color=yes --project=test -e '
-              using Pkg;
+          rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml \
+                LocalPreferences.toml test/LocalPreferences.toml
+
+          if [[ -n "${{ steps.legate-branch.outputs.branch }}" ]]; then
+            julia --color=yes --project=. -e '
+                using Pkg
+                Pkg.develop(PackageSpec(path = joinpath(ENV["LEGATE_CHECKOUT"], "lib/LegatePreferences")))
+                Pkg.develop(PackageSpec(path = ENV["LEGATE_CHECKOUT"]))
+                using LegatePreferences; LegatePreferences.use_developer_mode()
+                Pkg.build("Legate")
+            '
+          fi
+
+          julia --color=yes --project=. -e '
+              using Pkg
               Pkg.develop(PackageSpec(path = "lib/CNPreferences"))
-              using CNPreferences; CNPreferences.use_developer_mode();
+              using CNPreferences; CNPreferences.use_developer_mode()
+              Pkg.build("cuNumeric")
           '
-          julia --color=yes -e 'using Pkg; Pkg.build("cuNumeric")'
 
-      - name: Perform Test
-        run: |
-          GPUTESTS=0 julia --color=yes -e 'using Pkg; Pkg.test("cuNumeric")'
+          cp LocalPreferences.toml test/LocalPreferences.toml
+
+          julia --color=yes --project=. -e 'using Pkg; Pkg.test("cuNumeric")'

deps/build.jl

@@ -19,7 +19,12 @@
 
 using Pkg
 using Preferences
+
+# The build only needs Legate's paths/tooling, not a running runtime.
+# Setting this env prevents a segfault on Julia 1.12
+ENV["LEGATE_SKIP_RUNTIME"] = "true"
 using Legate
+
 using CNPreferences
 using CUDACore: CUDACore
 

src/cuNumeric.jl

@@ -191,7 +191,7 @@ end
 getargv(a::ArgcArgv) = Base.unsafe_convert(CxxPtr{CxxPtr{CxxChar}}, a.argv)
 
 function my_on_exit()
-    # @info "Cleaning Up cuNumeric"
+    return drain_pending_frees!()   # flush before Legate tears down
 end
 
 global cuNumeric_config_str::String = ""
@@ -215,6 +215,8 @@ function _start_runtime()
     # AA = ArgcArgv([Base.julia_cmd()[1]])
     cuNumeric.initialize_cunumeric(AA.argc, getargv(AA))
 
+    _init_deferred_free!()   # record launch thread for deferred frees (memory.jl)
+
     # setup /src/memory.jl
     cuNumeric.init_gc!()
 

src/memory.jl

@@ -1,4 +1,50 @@
-using Base.Threads: Atomic, atomic_add!, atomic_sub!, atomic_xchg!
+using Base.Threads: Atomic, atomic_add!, atomic_sub!, atomic_xchg!, SpinLock
+
+# Legate only permits handle destruction on the launch thread, but GC finalizers can
+# run on another thread (e.g. 1.12's interactive thread), so they enqueue here and the
+# launch thread drains later. Presized buffers keep the finalizer enqueue alloc-free.
+const _RUNTIME_TID = Ref{Int}(0)
+const _free_lock = SpinLock()
+const _free_queue = Ptr{Cvoid}[]   # producers (finalizers) push
+const _free_drain = Ptr{Cvoid}[]   # launch thread swaps into here to free
+
+function _init_deferred_free!()
+    _RUNTIME_TID[] = Threads.threadid()
+    sizehint!(_free_queue, 1 << 16)
+    sizehint!(_free_drain, 1 << 16)
+    return nothing
+end
+
+# Runs in finalizers on any thread: no Legate call, no block, no steady-state alloc.
+@inline function _enqueue_free!(ptr::Ptr{Cvoid})
+    ptr == C_NULL && return nothing
+    lock(_free_lock)
+    push!(_free_queue, ptr)
+    unlock(_free_lock)
+    return nothing
+end
+
+@doc"""
+    drain_pending_frees!()
+
+Destroy NDArray handles queued by finalizers. No-op off the launch thread. Called
+automatically from the op/allocation path, so user code rarely needs it.
+"""
+function drain_pending_frees!()
+    Threads.threadid() == _RUNTIME_TID[] || return nothing
+    isempty(_free_queue) && return nothing
+
+    lock(_free_lock)
+    append!(_free_drain, _free_queue)
+    empty!(_free_queue)
+    unlock(_free_lock)
+
+    for ptr in _free_drain
+        nda_destroy_array(ptr)
+    end
+    empty!(_free_drain)
+    return nothing
+end
 
 query_total_device_memory() = ccall((:nda_query_total_device_memory, libnda),
     Int64, ())
@@ -118,6 +164,7 @@ end
 
 function _collect!(full::Bool)
     GC.gc(full)
+    drain_pending_frees!()   # free what GC just enqueued
     recalibrate_allocator!()
     atomic_xchg!(post_gc_device_bytes, current_device_bytes[])
     atomic_xchg!(post_gc_host_bytes, current_host_bytes[])

src/ndarray/detail/ndarray.jl

@@ -8,9 +8,15 @@ struct Slice
     stop::Int64
 end
 
+# All op/allocation submissions pass through here; flush queued frees first so they
+# land in program order relative to operations rather than at arbitrary GC points.
 macro task_scope(scope_name, body)
-    TASK_SCOPE_NAMES || return esc(body)
+    TASK_SCOPE_NAMES || return quote
+        drain_pending_frees!()
+        $(esc(body))
+    end
     return quote
+        drain_pending_frees!()
         Legate.with_scope($(esc(scope_name))) do
             return $(esc(body))
         end
@@ -52,7 +58,7 @@ mutable struct NDArray{T,N,PADDED,P} <: AbstractNDArray{T,N}
         nbytes = cuNumeric.nda_nbytes(ptr)
         cuNumeric.register_alloc!(nbytes)
         handle = new{T,N,false,Nothing}(ptr, nbytes, nothing, nothing)
-        finalizer(destroy!, handle)
+        finalizer(_finalize_ndarray!, handle)
         return handle
     end
 
@@ -61,11 +67,24 @@ mutable struct NDArray{T,N,PADDED,P} <: AbstractNDArray{T,N}
         nbytes = cuNumeric.nda_nbytes(ptr)
         cuNumeric.register_alloc!(nbytes)
         handle = new{T,N,false,P}(ptr, nbytes, nothing, parent)
-        finalizer(destroy!, handle)
+        finalizer(_finalize_ndarray!, handle)
         return handle
     end
 end
 
+# May run off the launch thread, so defer the Legate free to drain_pending_frees!.
+# Accounting is atomic and safe to do here immediately.
+function _finalize_ndarray!(arr::NDArray)
+    ptr = arr.ptr
+    ptr == C_NULL && return nothing
+    arr.ptr = Ptr{Cvoid}(0)
+    nbytes = arr.nbytes
+    arr.nbytes = 0
+    nbytes > 0 && register_free!(nbytes)
+    _enqueue_free!(ptr)
+    return nothing
+end
+
 @inline _is_ndarray_slice(arr::NDArray) = arr.parent isa NDArray
 
 """
@@ -562,7 +581,7 @@ function compare(
     end
 
     for CI in CartesianIndices(julia_array)
-        x = julia_array[CI];
+        x = julia_array[CI]
         y = arr[Tuple(CI)...]
         if !isapprox(x, y; atol=atol, rtol=rtol)
             return false
@@ -587,7 +606,7 @@ function compare(arr::NDArray{T,N}, arr2::NDArray{T,N}, atol::Real, rtol::Real)
 
     dims = shape(arr)
     for CI in CartesianIndices(dims)
-        x = arr[Tuple(CI)...];
+        x = arr[Tuple(CI)...]
         y = arr2[Tuple(CI)...]
         if !isapprox(x, y; atol=atol, rtol=rtol)
             return false