Very WIP: Architecture for robust cancellation

AGENTS.md

@@ -27,6 +27,20 @@ If you made changes to the runtime (any files in `src/`), you will need to rebui
 julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes
 depending on your changes.
 
+### Testing LLVM-related changes
+
+When making changes to LLVM passes or codegen, add `LLVM_ASSERTIONS=1` to `Make.user` to enable
+LLVM assertions. This helps catch IR verification errors early:
+
+```bash
+echo "LLVM_ASSERTIONS=1" >> Make.user
+```
+
+To run LLVM pass tests:
+```bash
+make -C test/llvmpasses <testname>.ll
+```
+
 After making changes, run static analysis checks:
   - First run `make -C src install-analysis-deps` to initialize dependencies (only needed once the first time).
   - Run `make -C src analyze-<filename> --output-sync -j8` (replace `<filename>` with the basename of any C or C++ file you modified, excluding headers).

Compiler/src/effects.jl

@@ -11,28 +11,32 @@ The output represents the state of different effect properties in the following
     - `+e` (green): `ALWAYS_TRUE`
     - `-e` (red): `ALWAYS_FALSE`
     - `?e` (yellow): `EFFECT_FREE_IF_INACCESSIBLEMEMONLY`
-3. `nothrow` (`n`):
+3. `reset_safe` (`re`):
+    - `+re` (green): `ALWAYS_TRUE`
+    - `-re` (red): `ALWAYS_FALSE`
+    - `?re` (yellow): `RESET_SAFE_IF_INACCESSIBLEMEMONLY`
+4. `nothrow` (`n`):
     - `+n` (green): `true`
     - `-n` (red): `false`
-4. `terminates` (`t`):
+5. `terminates` (`t`):
     - `+t` (green): `true`
     - `-t` (red): `false`
-5. `notaskstate` (`s`):
+6. `notaskstate` (`s`):
     - `+s` (green): `true`
     - `-s` (red): `false`
-6. `inaccessiblememonly` (`m`):
+7. `inaccessiblememonly` (`m`):
     - `+m` (green): `ALWAYS_TRUE`
     - `-m` (red): `ALWAYS_FALSE`
     - `?m` (yellow): `INACCESSIBLEMEM_OR_ARGMEMONLY`
-7. `noub` (`u`):
+8. `noub` (`u`):
     - `+u` (green): `true`
     - `-u` (red): `false`
     - `?u` (yellow): `NOUB_IF_NOINBOUNDS`
-8. `:nonoverlayed` (`o`):
+9. `:nonoverlayed` (`o`):
     - `+o` (green): `ALWAYS_TRUE`
     - `-o` (red): `ALWAYS_FALSE`
     - `?o` (yellow): `CONSISTENT_OVERLAY`
-9. `:nortcall` (`r`):
+10. `:nortcall` (`r`):
     - `+r` (green): `true`
     - `-r` (red): `false`
 """
@@ -62,6 +66,10 @@ following meanings:
     will not be refined anyway.
   * `EFFECT_FREE_IF_INACCESSIBLEMEMONLY`: the `:effect-free`-ness of this method can later be
     refined to `ALWAYS_TRUE` in a case when `:inaccessiblememonly` is proven.
+- `reset_safe::UInt8`
+  * The execution of this function may be interrupted and reset to an earlier cancellation
+    point at any point in the function. The interpretation is similar to `:effect_free`,
+    but has different guarantees.
 - `nothrow::Bool`: this method is guaranteed to not throw an exception.
   If the execution of this method may raise `MethodError`s and similar exceptions, then
   the method is not considered as `:nothrow`.
@@ -119,6 +127,7 @@ $(effects_key_string)
 struct Effects
     consistent::UInt8
     effect_free::UInt8
+    reset_safe::UInt8
     nothrow::Bool
     terminates::Bool
     notaskstate::Bool
@@ -129,6 +138,7 @@ struct Effects
     function Effects(
         consistent::UInt8,
         effect_free::UInt8,
+        reset_safe::UInt8,
         nothrow::Bool,
         terminates::Bool,
         notaskstate::Bool,
@@ -139,6 +149,7 @@ struct Effects
         return new(
             consistent,
             effect_free,
+            reset_safe,
             nothrow,
             terminates,
             notaskstate,
@@ -175,14 +186,18 @@ const NOUB_IF_NOINBOUNDS = 0x01 << 1
 # :nonoverlayed bits
 const CONSISTENT_OVERLAY = 0x01 << 1
 
-const EFFECTS_TOTAL   = Effects(ALWAYS_TRUE,  ALWAYS_TRUE,  true,  true,  true,  ALWAYS_TRUE,  ALWAYS_TRUE,  ALWAYS_TRUE, true)
-const EFFECTS_THROWS  = Effects(ALWAYS_TRUE,  ALWAYS_TRUE,  false, true,  true,  ALWAYS_TRUE,  ALWAYS_TRUE,  ALWAYS_TRUE, true)
-const EFFECTS_UNKNOWN = Effects(ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_TRUE, false) # unknown mostly, but it's not overlayed at least (e.g. it's not a call)
+# :reset_safe bits
+const RESET_SAFE_IF_INACCESSIBLEMEMONLY = 0x01 << 1
 
-function Effects(effects::Effects=Effects(
-    ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false);
+const EFFECTS_TOTAL   = Effects(ALWAYS_TRUE,  ALWAYS_TRUE,  ALWAYS_TRUE, true,  true,  true,  ALWAYS_TRUE,  ALWAYS_TRUE,  ALWAYS_TRUE, true)
+const EFFECTS_THROWS  = Effects(ALWAYS_TRUE,  ALWAYS_TRUE,  ALWAYS_TRUE, false, true,  true,  ALWAYS_TRUE,  ALWAYS_TRUE,  ALWAYS_TRUE, true)
+const EFFECTS_UNKNOWN = Effects(ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_TRUE, false) # unknown mostly, but it's not overlayed at least (e.g. it's not a call)
+const EFFECTS_MINIMAL = Effects(ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false)
+
+function Effects(effects::Effects=EFFECTS_MINIMAL;
     consistent::UInt8 = effects.consistent,
     effect_free::UInt8 = effects.effect_free,
+    reset_safe::UInt8 = effects.reset_safe,
     nothrow::Bool = effects.nothrow,
     terminates::Bool = effects.terminates,
     notaskstate::Bool = effects.notaskstate,
@@ -193,6 +208,7 @@ function Effects(effects::Effects=Effects(
     return Effects(
         consistent,
         effect_free,
+        reset_safe,
         nothrow,
         terminates,
         notaskstate,
@@ -225,6 +241,14 @@ function is_better_effects(new::Effects, old::Effects)
     elseif new.effect_free != old.effect_free
         return false
     end
+    if new.reset_safe == ALWAYS_TRUE
+        any_improved |= old.reset_safe != ALWAYS_TRUE
+    elseif new.reset_safe == RESET_SAFE_IF_INACCESSIBLEMEMONLY
+        old.reset_safe == ALWAYS_TRUE && return false
+        any_improved |= old.reset_safe != RESET_SAFE_IF_INACCESSIBLEMEMONLY
+    elseif new.reset_safe != old.reset_safe
+        return false
+    end
     if new.nothrow
         any_improved |= !old.nothrow
     elseif new.nothrow != old.nothrow
@@ -276,6 +300,7 @@ function merge_effects(old::Effects, new::Effects)
     return Effects(
         merge_effectbits(old.consistent, new.consistent),
         merge_effectbits(old.effect_free, new.effect_free),
+        merge_effectbits(old.reset_safe, new.reset_safe),
         merge_effectbits(old.nothrow, new.nothrow),
         merge_effectbits(old.terminates, new.terminates),
         merge_effectbits(old.notaskstate, new.notaskstate),
@@ -295,6 +320,7 @@ merge_effectbits(old::Bool, new::Bool) = old & new
 
 is_consistent(effects::Effects)          = effects.consistent === ALWAYS_TRUE
 is_effect_free(effects::Effects)         = effects.effect_free === ALWAYS_TRUE
+is_reset_safe(effects::Effects)          = effects.reset_safe === ALWAYS_TRUE
 is_nothrow(effects::Effects)             = effects.nothrow
 is_terminates(effects::Effects)          = effects.terminates
 is_notaskstate(effects::Effects)         = effects.notaskstate
@@ -331,6 +357,8 @@ is_consistent_if_inaccessiblememonly(effects::Effects) = !iszero(effects.consist
 
 is_effect_free_if_inaccessiblememonly(effects::Effects) = !iszero(effects.effect_free & EFFECT_FREE_IF_INACCESSIBLEMEMONLY)
 
+is_reset_safe_if_inaccessiblememonly(effects::Effects) = !iszero(effects.reset_safe & RESET_SAFE_IF_INACCESSIBLEMEMONLY)
+
 is_inaccessiblemem_or_argmemonly(effects::Effects) = effects.inaccessiblememonly === INACCESSIBLEMEM_OR_ARGMEMONLY
 
 is_consistent_overlay(effects::Effects) = effects.nonoverlayed === CONSISTENT_OVERLAY
@@ -345,13 +373,15 @@ function encode_effects(e::Effects)
            ((e.inaccessiblememonly % UInt32) << 8)  |
            ((e.noub                % UInt32) << 10) |
            ((e.nonoverlayed        % UInt32) << 12) |
-           ((e.nortcall            % UInt32) << 14)
+           ((e.nortcall            % UInt32) << 14) |
+           ((e.reset_safe          % UInt32) << 15)
 end
 
 function decode_effects(e::UInt32)
     return Effects(
         UInt8((e >> 0) & 0x07),
         UInt8((e >> 3) & 0x03),
+        UInt8((e >> 15) & 0x03),
         Bool((e >> 5) & 0x01),
         Bool((e >> 6) & 0x01),
         Bool((e >> 7) & 0x01),

Compiler/src/optimize.jl

@@ -39,6 +39,9 @@ const IR_FLAG_NOUB        = one(UInt32) << 10
 #const IR_FLAG_CONSISTENTOVERLAY = one(UInt32) << 12
 # This statement is :nortcall
 const IR_FLAG_NORTCALL = one(UInt32) << 13
+# This statement is proven :reset_safe
+const IR_FLAG_RESET_SAFE = one(UInt32) << 14
+# Reserved: one(UInt32) << 15 used for RSIIMO below
 # An optimization pass has updated this statement in a way that may
 # have exposed information that inference did not see. Re-running
 # inference on this statement may be profitable.
@@ -50,16 +53,18 @@ const IR_FLAG_UNUSED      = one(UInt32) << 17
 const IR_FLAG_EFIIMO      = one(UInt32) << 18
 # This statement is :inaccessiblememonly == INACCESSIBLEMEM_OR_ARGMEMONLY
 const IR_FLAG_INACCESSIBLEMEM_OR_ARGMEM = one(UInt32) << 19
+# This statement is :reset_safe == RESET_SAFE_IF_INACCESSIBLEMEMONLY
+const IR_FLAG_RSIIMO      = one(UInt32) << 20
 
 const NUM_IR_FLAGS = 3 # sync with julia.h
 
 const IR_FLAGS_EFFECTS =
     IR_FLAG_CONSISTENT | IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW |
-    IR_FLAG_TERMINATES | IR_FLAG_NOUB | IR_FLAG_NORTCALL
+    IR_FLAG_TERMINATES | IR_FLAG_NOUB | IR_FLAG_NORTCALL | IR_FLAG_RESET_SAFE
 
 const IR_FLAGS_REMOVABLE = IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW | IR_FLAG_TERMINATES
 
-const IR_FLAGS_NEEDS_EA = IR_FLAG_EFIIMO | IR_FLAG_INACCESSIBLEMEM_OR_ARGMEM
+const IR_FLAGS_NEEDS_EA = IR_FLAG_EFIIMO | IR_FLAG_INACCESSIBLEMEM_OR_ARGMEM | IR_FLAG_RSIIMO
 
 has_flag(curr::UInt32, flag::UInt32) = (curr & flag) == flag
 
@@ -79,6 +84,11 @@ function flags_for_effects(effects::Effects)
     elseif is_effect_free_if_inaccessiblememonly(effects)
         flags |= IR_FLAG_EFIIMO
     end
+    if is_reset_safe(effects)
+        flags |= IR_FLAG_RESET_SAFE
+    elseif is_reset_safe_if_inaccessiblememonly(effects)
+        flags |= IR_FLAG_RSIIMO
+    end
     if is_nothrow(effects)
         flags |= IR_FLAG_NOTHROW
     end

Compiler/src/ssair/show.jl

@@ -1062,7 +1062,7 @@ function show_ir(io::IO, compact::IncrementalCompact, config::IRShowConfig=defau
     finish_show_ir(io, uncompacted_cfg, config)
 end
 
-function effectbits_letter(effects::Effects, name::Symbol, suffix::Char)
+function effectbits_letter(effects::Effects, name::Symbol, suffix::Union{Char, String})
     ft = fieldtype(Effects, name)
     if ft === UInt8
         prefix = getfield(effects, name) === ALWAYS_TRUE ? '+' :
@@ -1094,6 +1094,8 @@ function Base.show(io::IO, e::Effects)
     print(io, ',')
     printstyled(io, effectbits_letter(e, :effect_free, 'e'); color=effectbits_color(e, :effect_free))
     print(io, ',')
+    printstyled(io, effectbits_letter(e, :reset_safe, "re"); color=effectbits_color(e, :reset_safe))
+    print(io, ',')
     printstyled(io, effectbits_letter(e, :nothrow,     'n'); color=effectbits_color(e, :nothrow))
     print(io, ',')
     printstyled(io, effectbits_letter(e, :terminates,  't'); color=effectbits_color(e, :terminates))

Compiler/src/tfuncs.jl

@@ -716,7 +716,7 @@ end
 @nospecs function pointerset_tfunc(𝕃::AbstractLattice, a, v, i, align)
     return a
 end
-@nospecs function atomic_fence_tfunc(𝕃::AbstractLattice, order)
+@nospecs function atomic_fence_tfunc(𝕃::AbstractLattice, order, syncscope)
     return Nothing
 end
 @nospecs function atomic_pointerref_tfunc(𝕃::AbstractLattice, a, order)
@@ -757,7 +757,7 @@ add_tfunc(add_ptr, 2, 2, pointerarith_tfunc, 1)
 add_tfunc(sub_ptr, 2, 2, pointerarith_tfunc, 1)
 add_tfunc(pointerref, 3, 3, pointerref_tfunc, 4)
 add_tfunc(pointerset, 4, 4, pointerset_tfunc, 5)
-add_tfunc(atomic_fence, 1, 1, atomic_fence_tfunc, 4)
+add_tfunc(atomic_fence, 2, 2, atomic_fence_tfunc, 4)
 add_tfunc(atomic_pointerref, 2, 2, atomic_pointerref_tfunc, 4)
 add_tfunc(atomic_pointerset, 3, 3, atomic_pointerset_tfunc, 5)
 add_tfunc(atomic_pointerswap, 3, 3, atomic_pointerswap_tfunc, 5)

NEWS.md

@@ -21,6 +21,10 @@ Command-line option changes
 Multi-threading changes
 -----------------------
 
+  - New functions `Threads.atomic_fence_heavy` and `Threads.atoimc_fence_light` provide support for
+    asymmetric atomic fences, speeding up atomic synchronization where one side of the synchronization
+    runs significantly less often than the other ([#60311]).
+
 Build system changes
 --------------------
 

base/Base.jl

@@ -371,6 +371,35 @@ function start_profile_listener()
     ccall(:jl_set_peek_cond, Cvoid, (Ptr{Cvoid},), cond.handle)
 end
 
+function sigint_listener(cond::AsyncCondition)
+    while _trywait(cond)
+        # The SIGINT handler should have set a cancellation request on the roottask
+        cr = @atomic :acquire roottask.cancellation_request
+        cr === nothing && continue
+        cancel!(roottask, cr)
+    end
+    nothing
+end
+
+function start_sigint_listener()
+    cond = AsyncCondition()
+    uv_unref(cond.handle)
+    t = errormonitor(Threads.@spawn(sigint_listener(cond)))
+    atexit() do
+        # destroy this callback when exiting
+        ccall(:jl_set_sigint_cond, Cvoid, (Ptr{Cvoid},), C_NULL)
+        # this will prompt any ongoing or pending event to flush also
+        close(cond)
+        # error-propagation is not needed, since the errormonitor will handle printing that better
+        t === current_task() || _wait(t)
+    end
+    finalizer(cond) do c
+        # if something goes south, still make sure we aren't keeping a reference in C to this
+        ccall(:jl_set_sigint_cond, Cvoid, (Ptr{Cvoid},), C_NULL)
+    end
+    ccall(:jl_set_sigint_cond, Cvoid, (Ptr{Cvoid},), cond.handle)
+end
+
 function __init__()
     # Base library init
     global _atexit_hooks_finished = false
@@ -394,6 +423,7 @@ function __init__()
         # triggering a profile via signals is not implemented on windows
         start_profile_listener()
     end
+    start_sigint_listener()
     _require_world_age[] = get_world_counter()
     # Prevent spawned Julia process from getting stuck waiting on Tracy to connect.
     delete!(ENV, "JULIA_WAIT_FOR_TRACY")

base/asyncevent.jl

@@ -165,7 +165,7 @@ function _trywait(t::Union{Timer, AsyncCondition})
     set = t.set
     if set
         # full barrier now for AsyncCondition
-        t isa Timer || Core.Intrinsics.atomic_fence(:acquire_release)
+        t isa Timer || Core.Intrinsics.atomic_fence(:acquire_release, :system)
     else
         if !isopen(t)
             set = t.set
@@ -183,7 +183,7 @@ function _trywait(t::Union{Timer, AsyncCondition})
                 set = t.set
                 if !set && t.handle != C_NULL # wait for set or handle, but not the isopen flag
                     iolock_end()
-                    set = wait(t.cond)
+                    set = wait(t.cond; waitee=t)
                     unlock(t.cond)
                     iolock_begin()
                     lock(t.cond)
@@ -199,8 +199,14 @@ function _trywait(t::Union{Timer, AsyncCondition})
     return set
 end
 
+cancel_wait!(t::Union{Timer, AsyncCondition}, @nospecialize(creq)) = false
+cancel_wait!(t::Union{Timer, AsyncCondition}, task::Task, @nospecialize(creq)) =
+    cancel_wait!(t.cond, task, creq, false; waitee=t)
+
 function wait(t::Union{Timer, AsyncCondition})
-    _trywait(t) || throw(EOFError())
+    ok = _trywait(t)
+    @cancel_check
+    ok || throw(EOFError())
     nothing
 end
 

base/atomics.jl

@@ -10,7 +10,7 @@ export
     atomic_add!, atomic_sub!,
     atomic_and!, atomic_nand!, atomic_or!, atomic_xor!,
     atomic_max!, atomic_min!,
-    atomic_fence
+    atomic_fence, atomic_fence_light, atomic_fence_heavy
 
 """
     Threads.Atomic{T}
@@ -329,4 +329,28 @@ fences should not be necessary in most cases.
 
 For further details, see LLVM's `fence` instruction.
 """
-atomic_fence() = Core.Intrinsics.atomic_fence(:sequentially_consistent)
+atomic_fence() = Core.Intrinsics.atomic_fence(:sequentially_consistent, :system)
+
+"""
+    Threads.atomic_fence_light()
+
+Insert the light side of an asymmetric sequential-consistency memory fence.
+Asymmetric memory fences are useful in scenarios where one side of the
+synchronization runs significantly less often than the other side. Use this
+function on the side that runs often and [`atomic_fence_heavy`](@ref) on the
+side that runs rarely.
+
+On supported operating systems and architectures this fence is cheaper than
+`Threads.atomic_fence()`, but synchronizes only with [`atomic_fence_heavy`](@ref)
+calls from other threads.
+"""
+atomic_fence_light() = Core.Intrinsics.atomic_fence(:sequentially_consistent, :singlethread)
+
+"""
+    Threads.atomic_fence_heavy()
+
+Insert the heavy side of an asymmetric sequential-consistency memory fence.
+Use this function on the side that runs rarely.
+See [`atomic_fence_light`](@ref) for more details.
+"""
+atomic_fence_heavy() = ccall(:jl_membarrier, Cvoid, ())