{#syntax#}@branchHint(hint: BranchHint) void{#endsyntax#}
+ Hints to the optimizer how likely a given branch of control flow is to be reached.
+{#syntax#}BranchHint{#endsyntax#} can be found with {#syntax#}@import("std").builtin.BranchHint{#endsyntax#}.
+This function is only valid as the first statement in a control flow branch, or the first statement in a function.
+ {#header_close#} + {#header_open|@breakpoint#}{#syntax#}@breakpoint() void{#endsyntax#}
@@ -5252,15 +5259,6 @@ fn cmpxchgWeakButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_val
{#header_close#} - {#header_open|@setCold#} -{#syntax#}@setCold(comptime is_cold: bool) void{#endsyntax#}
- - Tells the optimizer that the current function is (or is not) rarely called. - - This function is only valid within function scope. -
- {#header_close#} - {#header_open|@setEvalBranchQuota#}{#syntax#}@setEvalBranchQuota(comptime new_quota: u32) void{#endsyntax#}
diff --git a/doc/langref/test_functions.zig b/doc/langref/test_functions.zig index 0c0a136950ba..10e334793e74 100644 --- a/doc/langref/test_functions.zig +++ b/doc/langref/test_functions.zig @@ -27,9 +27,9 @@ const WINAPI: std.builtin.CallingConvention = if (native_arch == .x86) .Stdcall extern "kernel32" fn ExitProcess(exit_code: u32) callconv(WINAPI) noreturn; extern "c" fn atan2(a: f64, b: f64) f64; -// The @setCold builtin tells the optimizer that a function is rarely called. +// The @branchHint builtin can be used to tell the optimizer that a function is rarely called ("cold"). fn abort() noreturn { - @setCold(true); + @branchHint(.cold); while (true) {} } diff --git a/lib/c.zig b/lib/c.zig index 32f813c57b65..a4de4498a259 100644 --- a/lib/c.zig +++ b/lib/c.zig @@ -46,7 +46,7 @@ comptime { // Avoid dragging in the runtime safety mechanisms into this .o file, // unless we're trying to test this file. pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, _: ?usize) noreturn { - @setCold(true); + @branchHint(.cold); _ = error_return_trace; if (builtin.is_test) { std.debug.panic("{s}", .{msg}); diff --git a/lib/compiler/aro/aro/Driver/Filesystem.zig b/lib/compiler/aro/aro/Driver/Filesystem.zig index 3889ec73b367..a81f31375306 100644 --- a/lib/compiler/aro/aro/Driver/Filesystem.zig +++ b/lib/compiler/aro/aro/Driver/Filesystem.zig @@ -4,7 +4,7 @@ const builtin = @import("builtin"); const is_windows = builtin.os.tag == .windows; fn readFileFake(entries: []const Filesystem.Entry, path: []const u8, buf: []u8) ?[]const u8 { - @setCold(true); + @branchHint(.cold); for (entries) |entry| { if (mem.eql(u8, entry.path, path)) { const len = @min(entry.contents.len, buf.len); @@ -16,7 +16,7 @@ fn readFileFake(entries: []const Filesystem.Entry, path: []const u8, buf: []u8) } fn findProgramByNameFake(entries: []const Filesystem.Entry, name: []const u8, path: ?[]const u8, buf: []u8) ?[]const u8 { - @setCold(true); + @branchHint(.cold); if (mem.indexOfScalar(u8, name, '/') != null) { @memcpy(buf[0..name.len], name); return buf[0..name.len]; @@ -35,7 +35,7 @@ fn findProgramByNameFake(entries: []const Filesystem.Entry, name: []const u8, pa } fn canExecuteFake(entries: []const Filesystem.Entry, path: []const u8) bool { - @setCold(true); + @branchHint(.cold); for (entries) |entry| { if (mem.eql(u8, entry.path, path)) { return entry.executable; @@ -45,7 +45,7 @@ fn canExecuteFake(entries: []const Filesystem.Entry, path: []const u8) bool { } fn existsFake(entries: []const Filesystem.Entry, path: []const u8) bool { - @setCold(true); + @branchHint(.cold); var buf: [std.fs.max_path_bytes]u8 = undefined; var fib = std.heap.FixedBufferAllocator.init(&buf); const resolved = std.fs.path.resolvePosix(fib.allocator(), &.{path}) catch return false; diff --git a/lib/compiler/aro/aro/Parser.zig b/lib/compiler/aro/aro/Parser.zig index 1cb5e18934df..d8d6f9d71c7b 100644 --- a/lib/compiler/aro/aro/Parser.zig +++ b/lib/compiler/aro/aro/Parser.zig @@ -385,12 +385,12 @@ fn errExpectedToken(p: *Parser, expected: Token.Id, actual: Token.Id) Error { } pub fn errStr(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, str: []const u8) Compilation.Error!void { - @setCold(true); + @branchHint(.cold); return p.errExtra(tag, tok_i, .{ .str = str }); } pub fn errExtra(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, extra: Diagnostics.Message.Extra) Compilation.Error!void { - @setCold(true); + @branchHint(.cold); const tok = p.pp.tokens.get(tok_i); var loc = tok.loc; if (tok_i != 0 and tok.id == .eof) { @@ -407,12 +407,12 @@ pub fn errExtra(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, extra: Diag } pub fn errTok(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex) Compilation.Error!void { - @setCold(true); + @branchHint(.cold); return p.errExtra(tag, tok_i, .{ .none = {} }); } pub fn err(p: *Parser, tag: Diagnostics.Tag) Compilation.Error!void { - @setCold(true); + @branchHint(.cold); return p.errExtra(tag, p.tok_i, .{ .none = {} }); } @@ -638,7 +638,7 @@ fn pragma(p: *Parser) Compilation.Error!bool { /// Issue errors for top-level definitions whose type was never completed. fn diagnoseIncompleteDefinitions(p: *Parser) !void { - @setCold(true); + @branchHint(.cold); const node_slices = p.nodes.slice(); const tags = node_slices.items(.tag); diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index e056e80252e1..c09801096cc2 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -421,7 +421,7 @@ fn cliDiagnosticsToErrorBundle( gpa: std.mem.Allocator, diagnostics: *cli.Diagnostics, ) !ErrorBundle { - @setCold(true); + @branchHint(.cold); var bundle: ErrorBundle.Wip = undefined; try bundle.init(gpa); @@ -468,7 +468,7 @@ fn diagnosticsToErrorBundle( diagnostics: *Diagnostics, mappings: SourceMappings, ) !ErrorBundle { - @setCold(true); + @branchHint(.cold); var bundle: ErrorBundle.Wip = undefined; try bundle.init(gpa); @@ -559,7 +559,7 @@ fn flushErrorMessageIntoBundle(wip: *ErrorBundle.Wip, msg: ErrorBundle.ErrorMess } fn errorStringToErrorBundle(allocator: std.mem.Allocator, comptime format: []const u8, args: anytype) !ErrorBundle { - @setCold(true); + @branchHint(.cold); var bundle: ErrorBundle.Wip = undefined; try bundle.init(allocator); errdefer bundle.deinit(); @@ -574,7 +574,7 @@ fn aroDiagnosticsToErrorBundle( fail_msg: []const u8, comp: *aro.Compilation, ) !ErrorBundle { - @setCold(true); + @branchHint(.cold); var bundle: ErrorBundle.Wip = undefined; try bundle.init(gpa); diff --git a/lib/compiler_rt/common.zig b/lib/compiler_rt/common.zig index 552fc07efd08..1645a3951deb 100644 --- a/lib/compiler_rt/common.zig +++ b/lib/compiler_rt/common.zig @@ -72,7 +72,7 @@ pub const want_sparc_abi = builtin.cpu.arch.isSPARC(); pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, _: ?usize) noreturn { _ = error_return_trace; if (builtin.is_test) { - @setCold(true); + @branchHint(.cold); std.debug.panic("{s}", .{msg}); } else { unreachable; diff --git a/lib/std/Thread/Futex.zig b/lib/std/Thread/Futex.zig index c067368041d0..fe22fa201123 100644 --- a/lib/std/Thread/Futex.zig +++ b/lib/std/Thread/Futex.zig @@ -27,7 +27,7 @@ const atomic = std.atomic; /// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically /// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`. pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void { - @setCold(true); + @branchHint(.cold); Impl.wait(ptr, expect, null) catch |err| switch (err) { error.Timeout => unreachable, // null timeout meant to wait forever @@ -43,7 +43,7 @@ pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void { /// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically /// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`. pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) error{Timeout}!void { - @setCold(true); + @branchHint(.cold); // Avoid calling into the OS for no-op timeouts. if (timeout_ns == 0) { @@ -56,7 +56,7 @@ pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) er /// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`. pub fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void { - @setCold(true); + @branchHint(.cold); // Avoid calling into the OS if there's nothing to wake up. if (max_waiters == 0) { @@ -1048,7 +1048,7 @@ pub const Deadline = struct { /// - A spurious wake occurs. /// - The deadline expires; In which case `error.Timeout` is returned. pub fn wait(self: *Deadline, ptr: *const atomic.Value(u32), expect: u32) error{Timeout}!void { - @setCold(true); + @branchHint(.cold); // Check if we actually have a timeout to wait until. // If not just wait "forever". diff --git a/lib/std/Thread/Mutex.zig b/lib/std/Thread/Mutex.zig index 032c19b7dd63..be421c4c9401 100644 --- a/lib/std/Thread/Mutex.zig +++ b/lib/std/Thread/Mutex.zig @@ -169,7 +169,7 @@ const FutexImpl = struct { } fn lockSlow(self: *@This()) void { - @setCold(true); + @branchHint(.cold); // Avoid doing an atomic swap below if we already know the state is contended. // An atomic swap unconditionally stores which marks the cache-line as modified unnecessarily. diff --git a/lib/std/Thread/ResetEvent.zig b/lib/std/Thread/ResetEvent.zig index b7e575878081..cbc5a2a31c2d 100644 --- a/lib/std/Thread/ResetEvent.zig +++ b/lib/std/Thread/ResetEvent.zig @@ -107,7 +107,7 @@ const FutexImpl = struct { } fn waitUntilSet(self: *Impl, timeout: ?u64) error{Timeout}!void { - @setCold(true); + @branchHint(.cold); // Try to set the state from `unset` to `waiting` to indicate // to the set() thread that others are blocked on the ResetEvent. diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig index e1c90b34c8b0..6028db83a013 100644 --- a/lib/std/builtin.zig +++ b/lib/std/builtin.zig @@ -675,6 +675,25 @@ pub const ExternOptions = struct { is_thread_local: bool = false, }; +/// This data structure is used by the Zig language code generation and +/// therefore must be kept in sync with the compiler implementation. +pub const BranchHint = enum(u3) { + /// Equivalent to no hint given. + none, + /// This branch of control flow is more likely to be reached than its peers. + /// The optimizer should optimize for reaching it. + likely, + /// This branch of control flow is less likely to be reached than its peers. + /// The optimizer should optimize for not reaching it. + unlikely, + /// This branch of control flow is unlikely to *ever* be reached. + /// The optimizer may place it in a different page of memory to optimize other branches. + cold, + /// It is difficult to predict whether this branch of control flow will be reached. + /// The optimizer should avoid branching behavior with expensive mispredictions. + unpredictable, +}; + /// This enum is set by the compiler and communicates which compiler backend is /// used to produce machine code. /// Think carefully before deciding to observe this value. Nearly all code should @@ -760,7 +779,7 @@ else /// This function is used by the Zig language code generation and /// therefore must be kept in sync with the compiler implementation. pub fn default_panic(msg: []const u8, error_return_trace: ?*StackTrace, ret_addr: ?usize) noreturn { - @setCold(true); + @branchHint(.cold); // For backends that cannot handle the language features depended on by the // default panic handler, we have a simpler panic handler: @@ -877,27 +896,27 @@ pub fn checkNonScalarSentinel(expected: anytype, actual: @TypeOf(expected)) void } pub fn panicSentinelMismatch(expected: anytype, actual: @TypeOf(expected)) noreturn { - @setCold(true); + @branchHint(.cold); std.debug.panicExtra(null, @returnAddress(), "sentinel mismatch: expected {any}, found {any}", .{ expected, actual }); } pub fn panicUnwrapError(st: ?*StackTrace, err: anyerror) noreturn { - @setCold(true); + @branchHint(.cold); std.debug.panicExtra(st, @returnAddress(), "attempt to unwrap error: {s}", .{@errorName(err)}); } pub fn panicOutOfBounds(index: usize, len: usize) noreturn { - @setCold(true); + @branchHint(.cold); std.debug.panicExtra(null, @returnAddress(), "index out of bounds: index {d}, len {d}", .{ index, len }); } pub fn panicStartGreaterThanEnd(start: usize, end: usize) noreturn { - @setCold(true); + @branchHint(.cold); std.debug.panicExtra(null, @returnAddress(), "start index {d} is larger than end index {d}", .{ start, end }); } pub fn panicInactiveUnionField(active: anytype, wanted: @TypeOf(active)) noreturn { - @setCold(true); + @branchHint(.cold); std.debug.panicExtra(null, @returnAddress(), "access of union field '{s}' while field '{s}' is active", .{ @tagName(wanted), @tagName(active) }); } @@ -930,7 +949,7 @@ pub const panic_messages = struct { }; pub noinline fn returnError(st: *StackTrace) void { - @setCold(true); + @branchHint(.cold); @setRuntimeSafety(false); addErrRetTraceAddr(st, @returnAddress()); } diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 826978abe42d..4672f7ac0610 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -409,7 +409,7 @@ pub fn assertReadable(slice: []const volatile u8) void { } pub fn panic(comptime format: []const u8, args: anytype) noreturn { - @setCold(true); + @branchHint(.cold); panicExtra(@errorReturnTrace(), @returnAddress(), format, args); } @@ -422,7 +422,7 @@ pub fn panicExtra( comptime format: []const u8, args: anytype, ) noreturn { - @setCold(true); + @branchHint(.cold); const size = 0x1000; const trunc_msg = "(msg truncated)"; @@ -450,7 +450,7 @@ threadlocal var panic_stage: usize = 0; // `panicImpl` could be useful in implementing a custom panic handler which // calls the default handler (on supported platforms) pub fn panicImpl(trace: ?*const std.builtin.StackTrace, first_trace_addr: ?usize, msg: []const u8) noreturn { - @setCold(true); + @branchHint(.cold); if (enable_segfault_handler) { // If a segfault happens while panicking, we want it to actually segfault, not trigger diff --git a/lib/std/fmt/parse_float/convert_slow.zig b/lib/std/fmt/parse_float/convert_slow.zig index e4539937bdb3..9232bc8bce40 100644 --- a/lib/std/fmt/parse_float/convert_slow.zig +++ b/lib/std/fmt/parse_float/convert_slow.zig @@ -36,7 +36,7 @@ pub fn getShift(n: usize) usize { /// Note that this function needs a lot of stack space and is marked /// cold to hint against inlining into the caller. pub fn convertSlow(comptime T: type, s: []const u8) BiasedFp(T) { - @setCold(true); + @branchHint(.cold); const MantissaT = mantissaType(T); const min_exponent = -(1 << (math.floatExponentBits(T) - 1)) + 1; diff --git a/lib/std/hash/xxhash.zig b/lib/std/hash/xxhash.zig index 1d7c8399fc91..eb3d4ea2b916 100644 --- a/lib/std/hash/xxhash.zig +++ b/lib/std/hash/xxhash.zig @@ -593,7 +593,7 @@ pub const XxHash3 = struct { } fn hash3(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 { - @setCold(true); + @branchHint(.cold); std.debug.assert(input.len > 0 and input.len < 4); const flip: [2]u32 = @bitCast(secret[0..8].*); @@ -609,7 +609,7 @@ pub const XxHash3 = struct { } fn hash8(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 { - @setCold(true); + @branchHint(.cold); std.debug.assert(input.len >= 4 and input.len <= 8); const flip: [2]u64 = @bitCast(secret[8..24].*); @@ -625,7 +625,7 @@ pub const XxHash3 = struct { } fn hash16(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 { - @setCold(true); + @branchHint(.cold); std.debug.assert(input.len > 8 and input.len <= 16); const flip: [4]u64 = @bitCast(secret[24..56].*); @@ -641,7 +641,7 @@ pub const XxHash3 = struct { } fn hash128(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 { - @setCold(true); + @branchHint(.cold); std.debug.assert(input.len > 16 and input.len <= 128); var acc = XxHash64.prime_1 *% @as(u64, input.len); @@ -657,7 +657,7 @@ pub const XxHash3 = struct { } fn hash240(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 { - @setCold(true); + @branchHint(.cold); std.debug.assert(input.len > 128 and input.len <= 240); var acc = XxHash64.prime_1 *% @as(u64, input.len); @@ -676,7 +676,7 @@ pub const XxHash3 = struct { } noinline fn hashLong(seed: u64, input: []const u8) u64 { - @setCold(true); + @branchHint(.cold); std.debug.assert(input.len >= 240); const block_count = ((input.len - 1) / @sizeOf(Block)) * @sizeOf(Block); diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index 905d44c3687d..c0ae4596dcf6 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -1657,7 +1657,7 @@ pub fn HashMapUnmanaged( } fn grow(self: *Self, allocator: Allocator, new_capacity: Size, ctx: Context) Allocator.Error!void { - @setCold(true); + @branchHint(.cold); const new_cap = @max(new_capacity, minimal_capacity); assert(new_cap > self.capacity()); assert(std.math.isPowerOfTwo(new_cap)); diff --git a/lib/std/heap/WasmPageAllocator.zig b/lib/std/heap/WasmPageAllocator.zig index 0fb45b70d4ee..7cb85561d829 100644 --- a/lib/std/heap/WasmPageAllocator.zig +++ b/lib/std/heap/WasmPageAllocator.zig @@ -61,7 +61,7 @@ const FreeBlock = struct { const not_found = maxInt(usize); fn useRecycled(self: FreeBlock, num_pages: usize, log2_align: u8) usize { - @setCold(true); + @branchHint(.cold); for (self.data, 0..) |segment, i| { const spills_into_next = @as(i128, @bitCast(segment)) < 0; const has_enough_bits = @popCount(segment) >= num_pages; diff --git a/lib/std/log.zig b/lib/std/log.zig index b2c05112b029..1533161cbae4 100644 --- a/lib/std/log.zig +++ b/lib/std/log.zig @@ -171,7 +171,7 @@ pub fn scoped(comptime scope: @Type(.EnumLiteral)) type { comptime format: []const u8, args: anytype, ) void { - @setCold(true); + @branchHint(.cold); log(.err, scope, format, args); } diff --git a/lib/std/once.zig b/lib/std/once.zig index ee3a8b7a357c..326487df076d 100644 --- a/lib/std/once.zig +++ b/lib/std/once.zig @@ -25,7 +25,7 @@ pub fn Once(comptime f: fn () void) type { } fn callSlow(self: *@This()) void { - @setCold(true); + @branchHint(.cold); self.mutex.lock(); defer self.mutex.unlock(); diff --git a/lib/std/posix.zig b/lib/std/posix.zig index 665ea17788ac..fcd923ad9c1b 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -654,7 +654,7 @@ fn getRandomBytesDevURandom(buf: []u8) !void { /// it raises SIGABRT followed by SIGKILL and finally lo /// Invokes the current signal handler for SIGABRT, if any. pub fn abort() noreturn { - @setCold(true); + @branchHint(.cold); // MSVCRT abort() sometimes opens a popup window which is undesirable, so // even when linking libc on Windows we use our own abort implementation. // See https://github.com/ziglang/zig/issues/2071 for more details. diff --git a/lib/std/sort/pdq.zig b/lib/std/sort/pdq.zig index 61d78797c569..55bd17ae93e5 100644 --- a/lib/std/sort/pdq.zig +++ b/lib/std/sort/pdq.zig @@ -203,7 +203,7 @@ fn partitionEqual(a: usize, b: usize, pivot: usize, context: anytype) usize { /// /// returns `true` if the slice is sorted at the end. This function is `O(n)` worst-case. fn partialInsertionSort(a: usize, b: usize, context: anytype) bool { - @setCold(true); + @branchHint(.cold); // maximum number of adjacent out-of-order pairs that will get shifted const max_steps = 5; @@ -247,7 +247,7 @@ fn partialInsertionSort(a: usize, b: usize, context: anytype) bool { } fn breakPatterns(a: usize, b: usize, context: anytype) void { - @setCold(true); + @branchHint(.cold); const len = b - a; if (len < 8) return; diff --git a/lib/std/zig/AstGen.zig b/lib/std/zig/AstGen.zig index 3450e39cc6be..6353e3b3ecd2 100644 --- a/lib/std/zig/AstGen.zig +++ b/lib/std/zig/AstGen.zig @@ -811,18 +811,18 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE .builtin_call_two, .builtin_call_two_comma => { if (node_datas[node].lhs == 0) { const params = [_]Ast.Node.Index{}; - return builtinCall(gz, scope, ri, node, ¶ms); + return builtinCall(gz, scope, ri, node, ¶ms, false); } else if (node_datas[node].rhs == 0) { const params = [_]Ast.Node.Index{node_datas[node].lhs}; - return builtinCall(gz, scope, ri, node, ¶ms); + return builtinCall(gz, scope, ri, node, ¶ms, false); } else { const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs }; - return builtinCall(gz, scope, ri, node, ¶ms); + return builtinCall(gz, scope, ri, node, ¶ms, false); } }, .builtin_call, .builtin_call_comma => { const params = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs]; - return builtinCall(gz, scope, ri, node, params); + return builtinCall(gz, scope, ri, node, params, false); }, .call_one, @@ -1017,16 +1017,16 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE .block_two, .block_two_semicolon => { const statements = [2]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs }; if (node_datas[node].lhs == 0) { - return blockExpr(gz, scope, ri, node, statements[0..0]); + return blockExpr(gz, scope, ri, node, statements[0..0], .normal); } else if (node_datas[node].rhs == 0) { - return blockExpr(gz, scope, ri, node, statements[0..1]); + return blockExpr(gz, scope, ri, node, statements[0..1], .normal); } else { - return blockExpr(gz, scope, ri, node, statements[0..2]); + return blockExpr(gz, scope, ri, node, statements[0..2], .normal); } }, .block, .block_semicolon => { const statements = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs]; - return blockExpr(gz, scope, ri, node, statements); + return blockExpr(gz, scope, ri, node, statements, .normal); }, .enum_literal => return simpleStrTok(gz, ri, main_tokens[node], node, .enum_literal), .error_value => return simpleStrTok(gz, ri, node_datas[node].rhs, node, .error_value), @@ -1241,7 +1241,7 @@ fn suspendExpr( suspend_scope.suspend_node = node; defer suspend_scope.unstack(); - const body_result = try fullBodyExpr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node); + const body_result = try fullBodyExpr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node, .normal); if (!gz.refIsNoReturn(body_result)) { _ = try suspend_scope.addBreak(.break_inline, suspend_inst, .void_value); } @@ -1362,7 +1362,7 @@ fn fnProtoExpr( assert(param_type_node != 0); var param_gz = block_scope.makeSubBlock(scope); defer param_gz.unstack(); - const param_type = try fullBodyExpr(¶m_gz, scope, coerced_type_ri, param_type_node); + const param_type = try fullBodyExpr(¶m_gz, scope, coerced_type_ri, param_type_node, .normal); const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1); _ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node); const main_tokens = tree.nodes.items(.main_token); @@ -2040,13 +2040,13 @@ fn comptimeExpr( else stmts[0..2]; - const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true); + const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true, .normal); return rvalue(gz, ri, block_ref, node); }, .block, .block_semicolon => { const stmts = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs]; // Replace result location and copy back later - see above. - const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true); + const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true, .normal); return rvalue(gz, ri, block_ref, node); }, else => unreachable, @@ -2071,7 +2071,7 @@ fn comptimeExpr( else .none, }; - const block_result = try fullBodyExpr(&block_scope, scope, ty_only_ri, node); + const block_result = try fullBodyExpr(&block_scope, scope, ty_only_ri, node, .normal); if (!gz.refIsNoReturn(block_result)) { _ = try block_scope.addBreak(.@"break", block_inst, block_result); } @@ -2311,6 +2311,7 @@ fn fullBodyExpr( scope: *Scope, ri: ResultInfo, node: Ast.Node.Index, + block_kind: BlockKind, ) InnerError!Zir.Inst.Ref { const tree = gz.astgen.tree; const node_tags = tree.nodes.items(.tag); @@ -2340,21 +2341,24 @@ fn fullBodyExpr( // Labeled blocks are tricky - forwarding result location information properly is non-trivial, // plus if this block is exited with a `break_inline` we aren't allowed multiple breaks. This // case is rare, so just treat it as a normal expression and create a nested block. - return expr(gz, scope, ri, node); + return blockExpr(gz, scope, ri, node, statements, block_kind); } var sub_gz = gz.makeSubBlock(scope); - try blockExprStmts(&sub_gz, &sub_gz.base, statements); + try blockExprStmts(&sub_gz, &sub_gz.base, statements, block_kind); return rvalue(gz, ri, .void_value, node); } +const BlockKind = enum { normal, allow_branch_hint }; + fn blockExpr( gz: *GenZir, scope: *Scope, ri: ResultInfo, block_node: Ast.Node.Index, statements: []const Ast.Node.Index, + kind: BlockKind, ) InnerError!Zir.Inst.Ref { const astgen = gz.astgen; const tree = astgen.tree; @@ -2365,7 +2369,7 @@ fn blockExpr( if (token_tags[lbrace - 1] == .colon and token_tags[lbrace - 2] == .identifier) { - return labeledBlockExpr(gz, scope, ri, block_node, statements, false); + return labeledBlockExpr(gz, scope, ri, block_node, statements, false, kind); } if (!gz.is_comptime) { @@ -2380,7 +2384,7 @@ fn blockExpr( var block_scope = gz.makeSubBlock(scope); defer block_scope.unstack(); - try blockExprStmts(&block_scope, &block_scope.base, statements); + try blockExprStmts(&block_scope, &block_scope.base, statements, kind); if (!block_scope.endsWithNoReturn()) { // As our last action before the break, "pop" the error trace if needed @@ -2391,7 +2395,7 @@ fn blockExpr( try block_scope.setBlockBody(block_inst); } else { var sub_gz = gz.makeSubBlock(scope); - try blockExprStmts(&sub_gz, &sub_gz.base, statements); + try blockExprStmts(&sub_gz, &sub_gz.base, statements, kind); } return rvalue(gz, ri, .void_value, block_node); @@ -2436,6 +2440,7 @@ fn labeledBlockExpr( block_node: Ast.Node.Index, statements: []const Ast.Node.Index, force_comptime: bool, + block_kind: BlockKind, ) InnerError!Zir.Inst.Ref { const astgen = gz.astgen; const tree = astgen.tree; @@ -2476,7 +2481,7 @@ fn labeledBlockExpr( if (force_comptime) block_scope.is_comptime = true; defer block_scope.unstack(); - try blockExprStmts(&block_scope, &block_scope.base, statements); + try blockExprStmts(&block_scope, &block_scope.base, statements, block_kind); if (!block_scope.endsWithNoReturn()) { // As our last action before the return, "pop" the error trace if needed _ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node); @@ -2495,7 +2500,7 @@ fn labeledBlockExpr( } } -fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Node.Index) !void { +fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Node.Index, block_kind: BlockKind) !void { const astgen = gz.astgen; const tree = astgen.tree; const node_tags = tree.nodes.items(.tag); @@ -2509,7 +2514,7 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod var noreturn_src_node: Ast.Node.Index = 0; var scope = parent_scope; - for (statements) |statement| { + for (statements, 0..) |statement, stmt_idx| { if (noreturn_src_node != 0) { try astgen.appendErrorNodeNotes( statement, @@ -2524,6 +2529,10 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod }, ); } + const allow_branch_hint = switch (block_kind) { + .normal => false, + .allow_branch_hint => stmt_idx == 0, + }; var inner_node = statement; while (true) { switch (node_tags[inner_node]) { @@ -2567,6 +2576,30 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod .for_simple, .@"for", => _ = try forExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullFor(inner_node).?, true), + // These cases are here to allow branch hints. + .builtin_call_two, .builtin_call_two_comma => { + try emitDbgNode(gz, inner_node); + const ri: ResultInfo = .{ .rl = .none }; + const result = if (node_data[inner_node].lhs == 0) r: { + break :r try builtinCall(gz, scope, ri, inner_node, &.{}, allow_branch_hint); + } else if (node_data[inner_node].rhs == 0) r: { + break :r try builtinCall(gz, scope, ri, inner_node, &.{node_data[inner_node].lhs}, allow_branch_hint); + } else r: { + break :r try builtinCall(gz, scope, ri, inner_node, &.{ + node_data[inner_node].lhs, + node_data[inner_node].rhs, + }, allow_branch_hint); + }; + noreturn_src_node = try addEnsureResult(gz, result, inner_node); + }, + .builtin_call, .builtin_call_comma => { + try emitDbgNode(gz, inner_node); + const ri: ResultInfo = .{ .rl = .none }; + const params = tree.extra_data[node_data[inner_node].lhs..node_data[inner_node].rhs]; + const result = try builtinCall(gz, scope, ri, inner_node, params, allow_branch_hint); + noreturn_src_node = try addEnsureResult(gz, result, inner_node); + }, + else => noreturn_src_node = try unusedResultExpr(gz, scope, inner_node), // zig fmt: on } @@ -2827,7 +2860,7 @@ fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: As .fence, .set_float_mode, .set_align_stack, - .set_cold, + .branch_hint, => break :b true, else => break :b false, }, @@ -4155,7 +4188,7 @@ fn fnDecl( assert(param_type_node != 0); var param_gz = decl_gz.makeSubBlock(scope); defer param_gz.unstack(); - const param_type = try fullBodyExpr(¶m_gz, params_scope, coerced_type_ri, param_type_node); + const param_type = try fullBodyExpr(¶m_gz, params_scope, coerced_type_ri, param_type_node, .normal); const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1); _ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node); @@ -4277,7 +4310,7 @@ fn fnDecl( var ret_gz = decl_gz.makeSubBlock(params_scope); defer ret_gz.unstack(); const ret_ref: Zir.Inst.Ref = inst: { - const inst = try fullBodyExpr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type); + const inst = try fullBodyExpr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type, .normal); if (ret_gz.instructionsSlice().len == 0) { // In this case we will send a len=0 body which can be encoded more efficiently. break :inst inst; @@ -4352,7 +4385,7 @@ fn fnDecl( const lbrace_line = astgen.source_line - decl_gz.decl_line; const lbrace_column = astgen.source_column; - _ = try fullBodyExpr(&fn_gz, params_scope, .{ .rl = .none }, body_node); + _ = try fullBodyExpr(&fn_gz, params_scope, .{ .rl = .none }, body_node, .allow_branch_hint); try checkUsed(gz, &fn_gz.base, params_scope); if (!fn_gz.endsWithNoReturn()) { @@ -4553,20 +4586,20 @@ fn globalVarDecl( var align_gz = block_scope.makeSubBlock(scope); if (var_decl.ast.align_node != 0) { - const align_inst = try fullBodyExpr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node); + const align_inst = try fullBodyExpr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node, .normal); _ = try align_gz.addBreakWithSrcNode(.break_inline, decl_inst, align_inst, node); } var linksection_gz = align_gz.makeSubBlock(scope); if (var_decl.ast.section_node != 0) { - const linksection_inst = try fullBodyExpr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node); + const linksection_inst = try fullBodyExpr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node, .normal); _ = try linksection_gz.addBreakWithSrcNode(.break_inline, decl_inst, linksection_inst, node); } var addrspace_gz = linksection_gz.makeSubBlock(scope); if (var_decl.ast.addrspace_node != 0) { const addrspace_ty = try addrspace_gz.addBuiltinValue(var_decl.ast.addrspace_node, .address_space); - const addrspace_inst = try fullBodyExpr(&addrspace_gz, &addrspace_gz.base, .{ .rl = .{ .coerced_ty = addrspace_ty } }, var_decl.ast.addrspace_node); + const addrspace_inst = try fullBodyExpr(&addrspace_gz, &addrspace_gz.base, .{ .rl = .{ .coerced_ty = addrspace_ty } }, var_decl.ast.addrspace_node, .normal); _ = try addrspace_gz.addBreakWithSrcNode(.break_inline, decl_inst, addrspace_inst, node); } @@ -4623,7 +4656,7 @@ fn comptimeDecl( }; defer decl_block.unstack(); - const block_result = try fullBodyExpr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node); + const block_result = try fullBodyExpr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node, .normal); if (decl_block.isEmpty() or !decl_block.refIsNoReturn(block_result)) { _ = try decl_block.addBreak(.break_inline, decl_inst, .void_value); } @@ -4844,7 +4877,7 @@ fn testDecl( const lbrace_line = astgen.source_line - decl_block.decl_line; const lbrace_column = astgen.source_column; - const block_result = try fullBodyExpr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node); + const block_result = try fullBodyExpr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node, .normal); if (fn_block.isEmpty() or !fn_block.refIsNoReturn(block_result)) { // As our last action before the return, "pop" the error trace if needed @@ -6113,7 +6146,7 @@ fn orelseCatchExpr( break :blk &err_val_scope.base; }; - const else_result = try fullBodyExpr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs); + const else_result = try fullBodyExpr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs, .allow_branch_hint); if (!else_scope.endsWithNoReturn()) { // As our last action before the break, "pop" the error trace if needed if (do_err_trace) @@ -6281,7 +6314,7 @@ fn boolBinOp( var rhs_scope = gz.makeSubBlock(scope); defer rhs_scope.unstack(); - const rhs = try fullBodyExpr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs); + const rhs = try fullBodyExpr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs, .allow_branch_hint); if (!gz.refIsNoReturn(rhs)) { _ = try rhs_scope.addBreakWithSrcNode(.break_inline, bool_br, rhs, node_datas[node].rhs); } @@ -6425,7 +6458,7 @@ fn ifExpr( } }; - const then_result = try fullBodyExpr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node); + const then_result = try fullBodyExpr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node, .allow_branch_hint); try checkUsed(parent_gz, &then_scope.base, then_sub_scope); if (!then_scope.endsWithNoReturn()) { _ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, then_node); @@ -6467,7 +6500,7 @@ fn ifExpr( break :s &else_scope.base; } }; - const else_result = try fullBodyExpr(&else_scope, sub_scope, block_scope.break_result_info, else_node); + const else_result = try fullBodyExpr(&else_scope, sub_scope, block_scope.break_result_info, else_node, .allow_branch_hint); if (!else_scope.endsWithNoReturn()) { // As our last action before the break, "pop" the error trace if needed if (do_err_trace) @@ -6576,7 +6609,7 @@ fn whileExpr( } = c: { if (while_full.error_token) |_| { const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none }; - const err_union = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr); + const err_union = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr, .normal); const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err; break :c .{ .inst = err_union, @@ -6584,14 +6617,14 @@ fn whileExpr( }; } else if (while_full.payload_token) |_| { const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none }; - const optional = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr); + const optional = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr, .normal); const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null; break :c .{ .inst = optional, .bool_bit = try cond_scope.addUnNode(tag, optional, while_full.ast.cond_expr), }; } else { - const cond = try fullBodyExpr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr); + const cond = try fullBodyExpr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr, .normal); break :c .{ .inst = cond, .bool_bit = cond, @@ -6716,7 +6749,7 @@ fn whileExpr( continue_scope.instructions_top = continue_scope.instructions.items.len; { try emitDbgNode(&continue_scope, then_node); - const unused_result = try fullBodyExpr(&continue_scope, &continue_scope.base, .{ .rl = .none }, then_node); + const unused_result = try fullBodyExpr(&continue_scope, &continue_scope.base, .{ .rl = .none }, then_node, .allow_branch_hint); _ = try addEnsureResult(&continue_scope, unused_result, then_node); } try checkUsed(parent_gz, &then_scope.base, then_sub_scope); @@ -6762,7 +6795,7 @@ fn whileExpr( // control flow apply to outer loops; not this one. loop_scope.continue_block = .none; loop_scope.break_block = .none; - const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node); + const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node, .allow_branch_hint); if (is_statement) { _ = try addEnsureResult(&else_scope, else_result, else_node); } @@ -7030,7 +7063,7 @@ fn forExpr( break :blk capture_sub_scope; }; - const then_result = try fullBodyExpr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node); + const then_result = try fullBodyExpr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node, .allow_branch_hint); _ = try addEnsureResult(&then_scope, then_result, then_node); try checkUsed(parent_gz, &then_scope.base, then_sub_scope); @@ -7049,7 +7082,7 @@ fn forExpr( // control flow apply to outer loops; not this one. loop_scope.continue_block = .none; loop_scope.break_block = .none; - const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node); + const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node, .allow_branch_hint); if (is_statement) { _ = try addEnsureResult(&else_scope, else_result, else_node); } @@ -7526,7 +7559,7 @@ fn switchExprErrUnion( } const target_expr_node = case.ast.target_expr; - const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node); + const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node, .allow_branch_hint); // check capture_scope, not err_scope to avoid false positive unused error capture try checkUsed(parent_gz, &case_scope.base, err_scope.parent); const uses_err = err_scope.used != 0 or err_scope.discarded != 0; @@ -7987,7 +8020,7 @@ fn switchExpr( try case_scope.addDbgVar(.dbg_var_val, dbg_var_tag_name, dbg_var_tag_inst); } const target_expr_node = case.ast.target_expr; - const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node); + const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node, .allow_branch_hint); try checkUsed(parent_gz, &case_scope.base, sub_scope); if (!parent_gz.refIsNoReturn(case_result)) { _ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node); @@ -9155,6 +9188,7 @@ fn builtinCall( ri: ResultInfo, node: Ast.Node.Index, params: []const Ast.Node.Index, + allow_branch_hint: bool, ) InnerError!Zir.Inst.Ref { const astgen = gz.astgen; const tree = astgen.tree; @@ -9188,6 +9222,18 @@ fn builtinCall( return astgen.failNode(node, "'{s}' outside function scope", .{builtin_name}); switch (info.tag) { + .branch_hint => { + if (!allow_branch_hint) { + return astgen.failNode(node, "'@branchHint' must appear as the first statement in a function or conditional branch", .{}); + } + const hint_ty = try gz.addBuiltinValue(node, .branch_hint); + const hint_val = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = hint_ty } }, params[0]); + _ = try gz.addExtendedPayload(.branch_hint, Zir.Inst.UnNode{ + .node = gz.nodeIndexToRelative(node), + .operand = hint_val, + }); + return rvalue(gz, ri, .void_value, node); + }, .import => { const node_tags = tree.nodes.items(.tag); const operand_node = params[0]; @@ -9371,14 +9417,6 @@ fn builtinCall( }); return rvalue(gz, ri, .void_value, node); }, - .set_cold => { - const order = try expr(gz, scope, ri, params[0]); - _ = try gz.addExtendedPayload(.set_cold, Zir.Inst.UnNode{ - .node = gz.nodeIndexToRelative(node), - .operand = order, - }); - return rvalue(gz, ri, .void_value, node); - }, .src => { // Incorporate the source location into the source hash, so that @@ -10040,7 +10078,7 @@ fn cImport( defer block_scope.unstack(); const block_inst = try gz.makeBlockInst(.c_import, node); - const block_result = try fullBodyExpr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node); + const block_result = try fullBodyExpr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node, .normal); _ = try gz.addUnNode(.ensure_result_used, block_result, node); if (!gz.refIsNoReturn(block_result)) { _ = try block_scope.addBreak(.break_inline, block_inst, .void_value); @@ -10123,7 +10161,7 @@ fn callExpr( defer arg_block.unstack(); // `call_inst` is reused to provide the param type. - const arg_ref = try fullBodyExpr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node); + const arg_ref = try fullBodyExpr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node, .normal); _ = try arg_block.addBreakWithSrcNode(.break_inline, call_index, arg_ref, param_node); const body = arg_block.instructionsSlice(); @@ -11474,7 +11512,7 @@ fn appendErrorNodeNotes( args: anytype, notes: []const u32, ) Allocator.Error!void { - @setCold(true); + @branchHint(.cold); const string_bytes = &astgen.string_bytes; const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len); try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args); @@ -11565,7 +11603,7 @@ fn appendErrorTokNotesOff( args: anytype, notes: []const u32, ) !void { - @setCold(true); + @branchHint(.cold); const gpa = astgen.gpa; const string_bytes = &astgen.string_bytes; const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len); @@ -11602,7 +11640,7 @@ fn errNoteTokOff( comptime format: []const u8, args: anytype, ) Allocator.Error!u32 { - @setCold(true); + @branchHint(.cold); const string_bytes = &astgen.string_bytes; const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len); try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args); @@ -11621,7 +11659,7 @@ fn errNoteNode( comptime format: []const u8, args: anytype, ) Allocator.Error!u32 { - @setCold(true); + @branchHint(.cold); const string_bytes = &astgen.string_bytes; const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len); try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args); diff --git a/lib/std/zig/AstRlAnnotate.zig b/lib/std/zig/AstRlAnnotate.zig index e956ffd2a9d6..597baa2936ae 100644 --- a/lib/std/zig/AstRlAnnotate.zig +++ b/lib/std/zig/AstRlAnnotate.zig @@ -829,6 +829,10 @@ fn builtinCall(astrl: *AstRlAnnotate, block: ?*Block, ri: ResultInfo, node: Ast. } switch (info.tag) { .import => return false, + .branch_hint => { + _ = try astrl.expr(args[0], block, ResultInfo.type_only); + return false; + }, .compile_log, .TypeOf => { for (args) |arg_node| { _ = try astrl.expr(arg_node, block, ResultInfo.none); @@ -907,7 +911,6 @@ fn builtinCall(astrl: *AstRlAnnotate, block: ?*Block, ri: ResultInfo, node: Ast. .fence, .set_float_mode, .set_align_stack, - .set_cold, .type_info, .work_item_id, .work_group_size, diff --git a/lib/std/zig/BuiltinFn.zig b/lib/std/zig/BuiltinFn.zig index fc08f9eb85db..1da3ffb5a727 100644 --- a/lib/std/zig/BuiltinFn.zig +++ b/lib/std/zig/BuiltinFn.zig @@ -14,6 +14,7 @@ pub const Tag = enum { bit_offset_of, int_from_bool, bit_size_of, + branch_hint, breakpoint, disable_instrumentation, mul_add, @@ -82,7 +83,6 @@ pub const Tag = enum { return_address, select, set_align_stack, - set_cold, set_eval_branch_quota, set_float_mode, set_runtime_safety, @@ -256,6 +256,14 @@ pub const list = list: { .param_count = 1, }, }, + .{ + "@branchHint", + .{ + .tag = .branch_hint, + .param_count = 1, + .illegal_outside_function = true, + }, + }, .{ "@breakpoint", .{ @@ -744,14 +752,6 @@ pub const list = list: { .illegal_outside_function = true, }, }, - .{ - "@setCold", - .{ - .tag = .set_cold, - .param_count = 1, - .illegal_outside_function = true, - }, - }, .{ "@setEvalBranchQuota", .{ diff --git a/lib/std/zig/Parse.zig b/lib/std/zig/Parse.zig index c43868a7d4c7..6f557a0f5526 100644 --- a/lib/std/zig/Parse.zig +++ b/lib/std/zig/Parse.zig @@ -81,7 +81,7 @@ fn addExtra(p: *Parse, extra: anytype) Allocator.Error!Node.Index { } fn warnExpected(p: *Parse, expected_token: Token.Tag) error{OutOfMemory}!void { - @setCold(true); + @branchHint(.cold); try p.warnMsg(.{ .tag = .expected_token, .token = p.tok_i, @@ -90,12 +90,12 @@ fn warnExpected(p: *Parse, expected_token: Token.Tag) error{OutOfMemory}!void { } fn warn(p: *Parse, error_tag: AstError.Tag) error{OutOfMemory}!void { - @setCold(true); + @branchHint(.cold); try p.warnMsg(.{ .tag = error_tag, .token = p.tok_i }); } fn warnMsg(p: *Parse, msg: Ast.Error) error{OutOfMemory}!void { - @setCold(true); + @branchHint(.cold); switch (msg.tag) { .expected_semi_after_decl, .expected_semi_after_stmt, @@ -141,12 +141,12 @@ fn warnMsg(p: *Parse, msg: Ast.Error) error{OutOfMemory}!void { } fn fail(p: *Parse, tag: Ast.Error.Tag) error{ ParseError, OutOfMemory } { - @setCold(true); + @branchHint(.cold); return p.failMsg(.{ .tag = tag, .token = p.tok_i }); } fn failExpected(p: *Parse, expected_token: Token.Tag) error{ ParseError, OutOfMemory } { - @setCold(true); + @branchHint(.cold); return p.failMsg(.{ .tag = .expected_token, .token = p.tok_i, @@ -155,7 +155,7 @@ fn failExpected(p: *Parse, expected_token: Token.Tag) error{ ParseError, OutOfMe } fn failMsg(p: *Parse, msg: Ast.Error) error{ ParseError, OutOfMemory } { - @setCold(true); + @branchHint(.cold); try p.warnMsg(msg); return error.ParseError; } diff --git a/lib/std/zig/Zir.zig b/lib/std/zig/Zir.zig index 26f4c4780ff3..7b750f91b207 100644 --- a/lib/std/zig/Zir.zig +++ b/lib/std/zig/Zir.zig @@ -1553,7 +1553,7 @@ pub const Inst = struct { => false, .extended => switch (data.extended.opcode) { - .fence, .set_cold, .breakpoint, .disable_instrumentation => true, + .fence, .branch_hint, .breakpoint, .disable_instrumentation => true, else => false, }, }; @@ -1962,9 +1962,6 @@ pub const Inst = struct { /// Implement builtin `@setAlignStack`. /// `operand` is payload index to `UnNode`. set_align_stack, - /// Implements `@setCold`. - /// `operand` is payload index to `UnNode`. - set_cold, /// Implements the `@errorCast` builtin. /// `operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand. error_cast, @@ -2059,6 +2056,10 @@ pub const Inst = struct { /// `operand` is `src_node: i32`. /// `small` is an `Inst.BuiltinValue`. builtin_value, + /// Provide a `@branchHint` for the current block. + /// `operand` is payload index to `UnNode`. + /// `small` is unused. + branch_hint, pub const InstData = struct { opcode: Extended, @@ -3150,6 +3151,7 @@ pub const Inst = struct { export_options, extern_options, type_info, + branch_hint, // Values calling_convention_c, calling_convention_inline, @@ -3981,7 +3983,6 @@ fn findDeclsInner( .fence, .set_float_mode, .set_align_stack, - .set_cold, .error_cast, .await_nosuspend, .breakpoint, @@ -4005,6 +4006,7 @@ fn findDeclsInner( .closure_get, .field_parent_ptr, .builtin_value, + .branch_hint, => return, // `@TypeOf` has a body. diff --git a/src/Air.zig b/src/Air.zig index 5c559a4088e5..2e45024919ed 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -433,13 +433,18 @@ pub const Inst = struct { /// In the case of non-error, control flow proceeds to the next instruction /// after the `try`, with the result of this instruction being the unwrapped /// payload value, as if `unwrap_errunion_payload` was executed on the operand. + /// The error branch is considered to have a branch hint of `.unlikely`. /// Uses the `pl_op` field. Payload is `Try`. @"try", + /// Same as `try` except the error branch hint is `.cold`. + try_cold, /// Same as `try` except the operand is a pointer to an error union, and the /// result is a pointer to the payload. Result is as if `unwrap_errunion_payload_ptr` /// was executed on the operand. /// Uses the `ty_pl` field. Payload is `TryPtr`. try_ptr, + /// Same as `try_ptr` except the error branch hint is `.cold`. + try_ptr_cold, /// Notes the beginning of a source code statement and marks the line and column. /// Result type is always void. /// Uses the `dbg_stmt` field. @@ -1116,11 +1121,20 @@ pub const Call = struct { pub const CondBr = struct { then_body_len: u32, else_body_len: u32, + branch_hints: BranchHints, + pub const BranchHints = packed struct(u32) { + true: std.builtin.BranchHint, + false: std.builtin.BranchHint, + _: u26 = 0, + }; }; /// Trailing: -/// * 0. `Case` for each `cases_len` -/// * 1. the else body, according to `else_body_len`. +/// * 0. `BranchHint` for each `cases_len + 1`. bit-packed into `u32` +/// elems such that each `u32` contains up to 10x `BranchHint`. +/// LSBs are first case. Final hint is `else`. +/// * 1. `Case` for each `cases_len` +/// * 2. the else body, according to `else_body_len`. pub const SwitchBr = struct { cases_len: u32, else_body_len: u32, @@ -1380,6 +1394,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool) .ptr_add, .ptr_sub, .try_ptr, + .try_ptr_cold, => return datas[@intFromEnum(inst)].ty_pl.ty.toType(), .not, @@ -1500,7 +1515,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool) return air.typeOf(extra.lhs, ip); }, - .@"try" => { + .@"try", .try_cold => { const err_union_ty = air.typeOf(datas[@intFromEnum(inst)].pl_op.operand, ip); return Type.fromInterned(ip.indexToKey(err_union_ty.ip_index).error_union_type.payload_type); }, @@ -1524,9 +1539,8 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end inline for (fields) |field| { @field(result, field.name) = switch (field.type) { u32 => air.extra[i], - Inst.Ref => @as(Inst.Ref, @enumFromInt(air.extra[i])), - i32 => @as(i32, @bitCast(air.extra[i])), - InternPool.Index => @as(InternPool.Index, @enumFromInt(air.extra[i])), + InternPool.Index, Inst.Ref => @enumFromInt(air.extra[i]), + i32, CondBr.BranchHints => @bitCast(air.extra[i]), else => @compileError("bad field type: " ++ @typeName(field.type)), }; i += 1; @@ -1593,7 +1607,9 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { .cond_br, .switch_br, .@"try", + .try_cold, .try_ptr, + .try_ptr_cold, .dbg_stmt, .dbg_inline_block, .dbg_var_ptr, @@ -1796,4 +1812,91 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { }; } +pub const UnwrappedSwitch = struct { + air: *const Air, + operand: Inst.Ref, + cases_len: u32, + else_body_len: u32, + branch_hints_start: u32, + cases_start: u32, + + /// Asserts that `case_idx < us.cases_len`. + pub fn getHint(us: UnwrappedSwitch, case_idx: u32) std.builtin.BranchHint { + assert(case_idx < us.cases_len); + return us.getHintInner(case_idx); + } + pub fn getElseHint(us: UnwrappedSwitch) std.builtin.BranchHint { + return us.getHintInner(us.cases_len); + } + fn getHintInner(us: UnwrappedSwitch, idx: u32) std.builtin.BranchHint { + const bag = us.air.extra[us.branch_hints_start..][idx / 10]; + const bits: u3 = @truncate(bag >> @intCast(3 * (idx % 10))); + return @enumFromInt(bits); + } + + pub fn iterateCases(us: UnwrappedSwitch) CaseIterator { + return .{ + .air = us.air, + .cases_len = us.cases_len, + .else_body_len = us.else_body_len, + .next_case = 0, + .extra_index = us.cases_start, + }; + } + pub const CaseIterator = struct { + air: *const Air, + cases_len: u32, + else_body_len: u32, + next_case: u32, + extra_index: u32, + + pub fn next(it: *CaseIterator) ?Case { + if (it.next_case == it.cases_len) return null; + const idx = it.next_case; + it.next_case += 1; + + const extra = it.air.extraData(SwitchBr.Case, it.extra_index); + var extra_index = extra.end; + const items: []const Inst.Ref = @ptrCast(it.air.extra[extra_index..][0..extra.data.items_len]); + extra_index += items.len; + const body: []const Inst.Index = @ptrCast(it.air.extra[extra_index..][0..extra.data.body_len]); + extra_index += body.len; + it.extra_index = @intCast(extra_index); + + return .{ + .idx = idx, + .items = items, + .body = body, + }; + } + /// Only valid to call once all cases have been iterated, i.e. `next` returns `null`. + /// Returns the body of the "default" (`else`) case. + pub fn elseBody(it: *CaseIterator) []const Inst.Index { + assert(it.next_case == it.cases_len); + return @ptrCast(it.air.extra[it.extra_index..][0..it.else_body_len]); + } + pub const Case = struct { + idx: u32, + items: []const Inst.Ref, + body: []const Inst.Index, + }; + }; +}; + +pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch { + const inst = air.instructions.get(@intFromEnum(switch_inst)); + assert(inst.tag == .switch_br); + const pl_op = inst.data.pl_op; + const extra = air.extraData(SwitchBr, pl_op.payload); + const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable; + return .{ + .air = air, + .operand = pl_op.operand, + .cases_len = extra.data.cases_len, + .else_body_len = extra.data.else_body_len, + .branch_hints_start = @intCast(extra.end), + .cases_start = @intCast(extra.end + hint_bag_count), + }; +} + pub const typesFullyResolved = @import("Air/types_resolved.zig").typesFullyResolved; diff --git a/src/Air/types_resolved.zig b/src/Air/types_resolved.zig index 4b92a3a94fcd..f51e4c2aea55 100644 --- a/src/Air/types_resolved.zig +++ b/src/Air/types_resolved.zig @@ -344,7 +344,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkRef(data.pl_op.operand, zcu)) return false; }, - .@"try" => { + .@"try", .try_cold => { const extra = air.extraData(Air.Try, data.pl_op.payload); if (!checkRef(data.pl_op.operand, zcu)) return false; if (!checkBody( @@ -354,7 +354,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { )) return false; }, - .try_ptr => { + .try_ptr, .try_ptr_cold => { const extra = air.extraData(Air.TryPtr, data.ty_pl.payload); if (!checkType(data.ty_pl.ty.toType(), zcu)) return false; if (!checkRef(extra.data.ptr, zcu)) return false; @@ -381,27 +381,14 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { }, .switch_br => { - const extra = air.extraData(Air.SwitchBr, data.pl_op.payload); - if (!checkRef(data.pl_op.operand, zcu)) return false; - var extra_index = extra.end; - for (0..extra.data.cases_len) |_| { - const case = air.extraData(Air.SwitchBr.Case, extra_index); - extra_index = case.end; - const items: []const Air.Inst.Ref = @ptrCast(air.extra[extra_index..][0..case.data.items_len]); - extra_index += case.data.items_len; - for (items) |item| if (!checkRef(item, zcu)) return false; - if (!checkBody( - air, - @ptrCast(air.extra[extra_index..][0..case.data.body_len]), - zcu, - )) return false; - extra_index += case.data.body_len; + const switch_br = air.unwrapSwitch(inst); + if (!checkRef(switch_br.operand, zcu)) return false; + var it = switch_br.iterateCases(); + while (it.next()) |case| { + for (case.items) |item| if (!checkRef(item, zcu)) return false; + if (!checkBody(air, case.body, zcu)) return false; } - if (!checkBody( - air, - @ptrCast(air.extra[extra_index..][0..extra.data.else_body_len]), - zcu, - )) return false; + if (!checkBody(air, it.elseBody(), zcu)) return false; }, .assembly => { diff --git a/src/Compilation.zig b/src/Compilation.zig index dc7d0ba92562..f1597091a9d3 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -5787,7 +5787,7 @@ fn failCObj( comptime format: []const u8, args: anytype, ) SemaError { - @setCold(true); + @branchHint(.cold); const diag_bundle = blk: { const diag_bundle = try comp.gpa.create(CObject.Diag.Bundle); diag_bundle.* = .{}; @@ -5811,7 +5811,7 @@ fn failCObjWithOwnedDiagBundle( c_object: *CObject, diag_bundle: *CObject.Diag.Bundle, ) SemaError { - @setCold(true); + @branchHint(.cold); assert(diag_bundle.diags.len > 0); { comp.mutex.lock(); @@ -5827,7 +5827,7 @@ fn failCObjWithOwnedDiagBundle( } fn failWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, comptime format: []const u8, args: anytype) SemaError { - @setCold(true); + @branchHint(.cold); var bundle: ErrorBundle.Wip = undefined; try bundle.init(comp.gpa); errdefer bundle.deinit(); @@ -5854,7 +5854,7 @@ fn failWin32ResourceWithOwnedBundle( win32_resource: *Win32Resource, err_bundle: ErrorBundle, ) SemaError { - @setCold(true); + @branchHint(.cold); { comp.mutex.lock(); defer comp.mutex.unlock(); diff --git a/src/InternPool.zig b/src/InternPool.zig index 83732a29f6ba..7d5e943b7ef6 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -2121,6 +2121,17 @@ pub const Key = union(enum) { @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release); } + pub fn setBranchHint(func: Func, ip: *InternPool, hint: std.builtin.BranchHint) void { + const extra_mutex = &ip.getLocal(func.tid).mutate.extra.mutex; + extra_mutex.lock(); + defer extra_mutex.unlock(); + + const analysis_ptr = func.analysisPtr(ip); + var analysis = analysis_ptr.*; + analysis.branch_hint = hint; + @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release); + } + /// Returns a pointer that becomes invalid after any additions to the `InternPool`. fn zirBodyInstPtr(func: Func, ip: *InternPool) *TrackedInst.Index { const extra = ip.getLocalShared(func.tid).extra.acquire(); @@ -5575,7 +5586,7 @@ pub const Tag = enum(u8) { /// to be part of the type of the function. pub const FuncAnalysis = packed struct(u32) { state: State, - is_cold: bool, + branch_hint: std.builtin.BranchHint, is_noinline: bool, calls_or_awaits_errorable_fn: bool, stack_alignment: Alignment, @@ -5583,7 +5594,7 @@ pub const FuncAnalysis = packed struct(u32) { inferred_error_set: bool, disable_instrumentation: bool, - _: u19 = 0, + _: u17 = 0, pub const State = enum(u2) { /// The runtime function has never been referenced. @@ -8636,7 +8647,7 @@ pub fn getFuncDecl( const func_decl_extra_index = addExtraAssumeCapacity(extra, Tag.FuncDecl{ .analysis = .{ .state = .unreferenced, - .is_cold = false, + .branch_hint = .none, .is_noinline = key.is_noinline, .calls_or_awaits_errorable_fn = false, .stack_alignment = .none, @@ -8740,7 +8751,7 @@ pub fn getFuncDeclIes( const func_decl_extra_index = addExtraAssumeCapacity(extra, Tag.FuncDecl{ .analysis = .{ .state = .unreferenced, - .is_cold = false, + .branch_hint = .none, .is_noinline = key.is_noinline, .calls_or_awaits_errorable_fn = false, .stack_alignment = .none, @@ -8932,7 +8943,7 @@ pub fn getFuncInstance( const func_extra_index = addExtraAssumeCapacity(extra, Tag.FuncInstance{ .analysis = .{ .state = .unreferenced, - .is_cold = false, + .branch_hint = .none, .is_noinline = arg.is_noinline, .calls_or_awaits_errorable_fn = false, .stack_alignment = .none, @@ -9032,7 +9043,7 @@ pub fn getFuncInstanceIes( const func_extra_index = addExtraAssumeCapacity(extra, Tag.FuncInstance{ .analysis = .{ .state = .unreferenced, - .is_cold = false, + .branch_hint = .none, .is_noinline = arg.is_noinline, .calls_or_awaits_errorable_fn = false, .stack_alignment = .none, @@ -11853,18 +11864,6 @@ pub fn funcSetDisableInstrumentation(ip: *InternPool, func: Index) void { @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release); } -pub fn funcSetCold(ip: *InternPool, func: Index, is_cold: bool) void { - const unwrapped_func = func.unwrap(ip); - const extra_mutex = &ip.getLocal(unwrapped_func.tid).mutate.extra.mutex; - extra_mutex.lock(); - defer extra_mutex.unlock(); - - const analysis_ptr = ip.funcAnalysisPtr(func); - var analysis = analysis_ptr.*; - analysis.is_cold = is_cold; - @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release); -} - pub fn funcZirBodyInst(ip: *const InternPool, func: Index) TrackedInst.Index { const unwrapped_func = func.unwrap(ip); const item = unwrapped_func.getItem(ip); diff --git a/src/Liveness.zig b/src/Liveness.zig index b75fc402dd3e..f26ea9a7a7d7 100644 --- a/src/Liveness.zig +++ b/src/Liveness.zig @@ -658,10 +658,10 @@ pub fn categorizeOperand( return .complex; }, - .@"try" => { + .@"try", .try_cold => { return .complex; }, - .try_ptr => { + .try_ptr, .try_ptr_cold => { return .complex; }, .loop => { @@ -1254,8 +1254,8 @@ fn analyzeInst( }, .loop => return analyzeInstLoop(a, pass, data, inst), - .@"try" => return analyzeInstCondBr(a, pass, data, inst, .@"try"), - .try_ptr => return analyzeInstCondBr(a, pass, data, inst, .try_ptr), + .@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"), + .try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr), .cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br), .switch_br => return analyzeInstSwitchBr(a, pass, data, inst), @@ -1674,21 +1674,18 @@ fn analyzeInstSwitchBr( const inst_datas = a.air.instructions.items(.data); const pl_op = inst_datas[@intFromEnum(inst)].pl_op; const condition = pl_op.operand; - const switch_br = a.air.extraData(Air.SwitchBr, pl_op.payload); + const switch_br = a.air.unwrapSwitch(inst); const gpa = a.gpa; - const ncases = switch_br.data.cases_len; + const ncases = switch_br.cases_len; switch (pass) { .loop_analysis => { - var air_extra_index: usize = switch_br.end; - for (0..ncases) |_| { - const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index); - const case_body: []const Air.Inst.Index = @ptrCast(a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]); - air_extra_index = case.end + case.data.items_len + case_body.len; - try analyzeBody(a, pass, data, case_body); + var it = switch_br.iterateCases(); + while (it.next()) |case| { + try analyzeBody(a, pass, data, case.body); } { // else - const else_body: []const Air.Inst.Index = @ptrCast(a.air.extra[air_extra_index..][0..switch_br.data.else_body_len]); + const else_body = it.elseBody(); try analyzeBody(a, pass, data, else_body); } }, @@ -1706,16 +1703,13 @@ fn analyzeInstSwitchBr( @memset(case_live_sets, .{}); defer for (case_live_sets) |*live_set| live_set.deinit(gpa); - var air_extra_index: usize = switch_br.end; - for (case_live_sets[0..ncases]) |*live_set| { - const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index); - const case_body: []const Air.Inst.Index = @ptrCast(a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]); - air_extra_index = case.end + case.data.items_len + case_body.len; - try analyzeBody(a, pass, data, case_body); - live_set.* = data.live_set.move(); + var case_it = switch_br.iterateCases(); + while (case_it.next()) |case| { + try analyzeBody(a, pass, data, case.body); + case_live_sets[case.idx] = data.live_set.move(); } { // else - const else_body: []const Air.Inst.Index = @ptrCast(a.air.extra[air_extra_index..][0..switch_br.data.else_body_len]); + const else_body = case_it.elseBody(); try analyzeBody(a, pass, data, else_body); case_live_sets[ncases] = data.live_set.move(); } diff --git a/src/Liveness/Verify.zig b/src/Liveness/Verify.zig index 7a9959481a30..bcd60d72c8d9 100644 --- a/src/Liveness/Verify.zig +++ b/src/Liveness/Verify.zig @@ -374,7 +374,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { }, // control flow - .@"try" => { + .@"try", .try_cold => { const pl_op = data[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Try, pl_op.payload); const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); @@ -396,7 +396,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { try self.verifyInst(inst); }, - .try_ptr => { + .try_ptr, .try_ptr_cold => { const ty_pl = data[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.TryPtr, ty_pl.payload); const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); @@ -509,44 +509,33 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { try self.verifyInst(inst); }, .switch_br => { - const pl_op = data[@intFromEnum(inst)].pl_op; - const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload); - var extra_index = switch_br.end; - var case_i: u32 = 0; + const switch_br = self.air.unwrapSwitch(inst); const switch_br_liveness = try self.liveness.getSwitchBr( self.gpa, inst, - switch_br.data.cases_len + 1, + switch_br.cases_len + 1, ); defer self.gpa.free(switch_br_liveness.deaths); - try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0)); + try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0)); var live = self.live.move(); defer live.deinit(self.gpa); - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items = @as( - []const Air.Inst.Ref, - @ptrCast(self.air.extra[case.end..][0..case.data.items_len]), - ); - const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + items.len + case_body.len; - + var it = switch_br.iterateCases(); + while (it.next()) |case| { self.live.deinit(self.gpa); self.live = try live.clone(self.gpa); - for (switch_br_liveness.deaths[case_i]) |death| try self.verifyDeath(inst, death); - try self.verifyBody(case_body); + for (switch_br_liveness.deaths[case.idx]) |death| try self.verifyDeath(inst, death); + try self.verifyBody(case.body); } - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]); + const else_body = it.elseBody(); if (else_body.len > 0) { self.live.deinit(self.gpa); self.live = try live.clone(self.gpa); - - for (switch_br_liveness.deaths[case_i]) |death| try self.verifyDeath(inst, death); + for (switch_br_liveness.deaths[switch_br.cases_len]) |death| try self.verifyDeath(inst, death); try self.verifyBody(else_body); } diff --git a/src/Sema.zig b/src/Sema.zig index 3752cefe3f04..6e8f13f1c0db 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -118,6 +118,10 @@ dependencies: std.AutoArrayHashMapUnmanaged(InternPool.Dependee, void) = .{}, /// by `analyzeCall`. allow_memoize: bool = true, +/// The `BranchHint` for the current branch of runtime control flow. +/// This state is on `Sema` so that `cold` hints can be propagated up through blocks with less special handling. +branch_hint: ?std.builtin.BranchHint = null, + const MaybeComptimeAlloc = struct { /// The runtime index of the `alloc` instruction. runtime_index: Value.RuntimeIndex, @@ -893,7 +897,12 @@ pub fn deinit(sema: *Sema) void { /// Performs semantic analysis of a ZIR body which is behind a runtime condition. If comptime /// control flow happens here, Sema will convert it to runtime control flow by introducing post-hoc /// blocks where necessary. -fn analyzeBodyRuntimeBreak(sema: *Sema, block: *Block, body: []const Zir.Inst.Index) !void { +/// Returns the branch hint for this branch. +fn analyzeBodyRuntimeBreak(sema: *Sema, block: *Block, body: []const Zir.Inst.Index) !std.builtin.BranchHint { + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint; + sema.branch_hint = null; + sema.analyzeBodyInner(block, body) catch |err| switch (err) { error.ComptimeBreak => { const zir_datas = sema.code.instructions.items(.data); @@ -903,6 +912,8 @@ fn analyzeBodyRuntimeBreak(sema: *Sema, block: *Block, body: []const Zir.Inst.In }, else => |e| return e, }; + + return sema.branch_hint orelse .none; } /// Semantically analyze a ZIR function body. It is guranteed by AstGen that such a body cannot @@ -1305,11 +1316,6 @@ fn analyzeBodyInner( i += 1; continue; }, - .set_cold => { - try sema.zirSetCold(block, extended); - i += 1; - continue; - }, .breakpoint => { if (!block.is_comptime) { _ = try block.addNoOp(.breakpoint); @@ -1327,6 +1333,11 @@ fn analyzeBodyInner( i += 1; continue; }, + .branch_hint => { + try sema.zirBranchHint(block, extended); + i += 1; + continue; + }, .value_placeholder => unreachable, // never appears in a body .field_parent_ptr => try sema.zirFieldParentPtr(block, extended), .builtin_value => try sema.zirBuiltinValue(extended), @@ -2466,7 +2477,7 @@ fn addFieldErrNote( comptime format: []const u8, args: anytype, ) !void { - @setCold(true); + @branchHint(.cold); const type_src = container_ty.srcLocOrNull(sema.pt.zcu) orelse return; const field_src: LazySrcLoc = .{ .base_node_inst = type_src.base_node_inst, @@ -2502,7 +2513,7 @@ pub fn fail( } pub fn failWithOwnedErrorMsg(sema: *Sema, block: ?*Block, err_msg: *Module.ErrorMsg) error{ AnalysisFail, OutOfMemory } { - @setCold(true); + @branchHint(.cold); const gpa = sema.gpa; const mod = sema.pt.zcu; @@ -5733,6 +5744,13 @@ fn zirPanic(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void if (block.is_comptime) { return sema.fail(block, src, "encountered @panic at comptime", .{}); } + + // We only apply the first hint in a branch. + // This allows user-provided hints to override implicit cold hints. + if (sema.branch_hint == null) { + sema.branch_hint = .cold; + } + try sema.panicWithMsg(block, src, coerced_msg, .@"@panic"); } @@ -6425,25 +6443,6 @@ fn zirSetAlignStack(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.Inst sema.allow_memoize = false; } -fn zirSetCold(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!void { - const pt = sema.pt; - const mod = pt.zcu; - const ip = &mod.intern_pool; - const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data; - const operand_src = block.builtinCallArgSrc(extra.node, 0); - const is_cold = try sema.resolveConstBool(block, operand_src, extra.operand, .{ - .needed_comptime_reason = "operand to @setCold must be comptime-known", - }); - // TODO: should `@setCold` apply to the parent in an inline call? - // See also #20642 and friends. - const func = switch (sema.owner.unwrap()) { - .func => |func| func, - .cau => return, // does nothing outside a function - }; - ip.funcSetCold(func, is_cold); - sema.allow_memoize = false; -} - fn zirDisableInstrumentation(sema: *Sema) CompileError!void { const pt = sema.pt; const mod = pt.zcu; @@ -6898,13 +6897,20 @@ fn popErrorReturnTrace( @typeInfo(Air.Block).Struct.fields.len + 1); // +1 for the sole .cond_br instruction in the .block const cond_br_inst: Air.Inst.Index = @enumFromInt(sema.air_instructions.len); - try sema.air_instructions.append(gpa, .{ .tag = .cond_br, .data = .{ .pl_op = .{ - .operand = is_non_error_inst, - .payload = sema.addExtraAssumeCapacity(Air.CondBr{ - .then_body_len = @intCast(then_block.instructions.items.len), - .else_body_len = @intCast(else_block.instructions.items.len), - }), - } } }); + try sema.air_instructions.append(gpa, .{ + .tag = .cond_br, + .data = .{ + .pl_op = .{ + .operand = is_non_error_inst, + .payload = sema.addExtraAssumeCapacity(Air.CondBr{ + .then_body_len = @intCast(then_block.instructions.items.len), + .else_body_len = @intCast(else_block.instructions.items.len), + // weight against error branch + .branch_hints = .{ .true = .likely, .false = .unlikely }, + }), + }, + }, + }); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items)); @@ -10961,6 +10967,11 @@ const SwitchProngAnalysis = struct { sema.code.instructions.items(.data)[@intFromEnum(spa.switch_block_inst)].pl_node.src_node, ); + // We can propagate `.cold` hints from this branch since it's comptime-known + // to be taken from the parent branch. + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null; + if (has_tag_capture) { const tag_ref = try spa.analyzeTagCapture(child_block, capture_src, inline_case_capture); sema.inst_map.putAssumeCapacity(spa.tag_capture_inst, tag_ref); @@ -10997,6 +11008,7 @@ const SwitchProngAnalysis = struct { /// Analyze a switch prong which may have peers at runtime. /// Uses `analyzeBodyRuntimeBreak`. Sets up captures as needed. + /// Returns the `BranchHint` for the prong. fn analyzeProngRuntime( spa: SwitchProngAnalysis, case_block: *Block, @@ -11014,7 +11026,7 @@ const SwitchProngAnalysis = struct { /// Whether this prong has an inline tag capture. If `true`, then /// `inline_case_capture` cannot be `.none`. has_tag_capture: bool, - ) CompileError!void { + ) CompileError!std.builtin.BranchHint { const sema = spa.sema; if (has_tag_capture) { @@ -11040,7 +11052,7 @@ const SwitchProngAnalysis = struct { if (sema.typeOf(capture_ref).isNoReturn(sema.pt.zcu)) { // No need to analyze any further, the prong is unreachable - return; + return .none; } sema.inst_map.putAssumeCapacity(spa.switch_block_inst, capture_ref); @@ -11309,10 +11321,17 @@ const SwitchProngAnalysis = struct { const prong_count = field_indices.len - in_mem_coercible.count(); - const estimated_extra = prong_count * 6; // 2 for Case, 1 item, probably 3 insts + const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints var cases_extra = try std.ArrayList(u32).initCapacity(sema.gpa, estimated_extra); defer cases_extra.deinit(); + { + // All branch hints are `.none`, so just add zero elems. + comptime assert(@intFromEnum(std.builtin.BranchHint.none) == 0); + const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable; + try cases_extra.appendNTimes(0, need_elems); + } + { // Non-bitcast cases var it = in_mem_coercible.iterator(.{ .kind = .unset }); @@ -11735,7 +11754,7 @@ fn zirSwitchBlockErrUnion(sema: *Sema, block: *Block, inst: Zir.Inst.Index) Comp sub_block.need_debug_scope = null; // this body is emitted regardless defer sub_block.instructions.deinit(gpa); - try sema.analyzeBodyRuntimeBreak(&sub_block, non_error_case.body); + const non_error_hint = try sema.analyzeBodyRuntimeBreak(&sub_block, non_error_case.body); const true_instructions = try sub_block.instructions.toOwnedSlice(gpa); defer gpa.free(true_instructions); @@ -11789,6 +11808,7 @@ fn zirSwitchBlockErrUnion(sema: *Sema, block: *Block, inst: Zir.Inst.Index) Comp .payload = sema.addExtraAssumeCapacity(Air.CondBr{ .then_body_len = @intCast(true_instructions.len), .else_body_len = @intCast(sub_block.instructions.items.len), + .branch_hints = .{ .true = non_error_hint, .false = .none }, }), } }, }); @@ -12493,6 +12513,9 @@ fn analyzeSwitchRuntimeBlock( var cases_extra = try std.ArrayListUnmanaged(u32).initCapacity(gpa, estimated_cases_extra); defer cases_extra.deinit(gpa); + var branch_hints = try std.ArrayListUnmanaged(std.builtin.BranchHint).initCapacity(gpa, scalar_cases_len); + defer branch_hints.deinit(gpa); + var case_block = child_block.makeSubBlock(); case_block.runtime_loop = null; case_block.runtime_cond = operand_src; @@ -12523,10 +12546,13 @@ fn analyzeSwitchRuntimeBlock( break :blk field_ty.zigTypeTag(mod) != .NoReturn; } else true; - if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) { - // nothing to do here - } else if (analyze_body) { - try spa.analyzeProngRuntime( + const prong_hint: std.builtin.BranchHint = if (err_set and + try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) + h: { + // nothing to do here. weight against error branch + break :h .unlikely; + } else if (analyze_body) h: { + break :h try spa.analyzeProngRuntime( &case_block, .normal, body, @@ -12539,10 +12565,12 @@ fn analyzeSwitchRuntimeBlock( if (info.is_inline) item else .none, info.has_tag_capture, ); - } else { + } else h: { _ = try case_block.addNoOp(.unreach); - } + break :h .none; + }; + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len cases_extra.appendAssumeCapacity(@intCast(case_block.instructions.items.len)); @@ -12552,6 +12580,7 @@ fn analyzeSwitchRuntimeBlock( var is_first = true; var prev_cond_br: Air.Inst.Index = undefined; + var prev_hint: std.builtin.BranchHint = undefined; var first_else_body: []const Air.Inst.Index = &.{}; defer gpa.free(first_else_body); var prev_then_body: []const Air.Inst.Index = &.{}; @@ -12613,7 +12642,7 @@ fn analyzeSwitchRuntimeBlock( } })); emit_bb = true; - try spa.analyzeProngRuntime( + const prong_hint = try spa.analyzeProngRuntime( &case_block, .normal, body, @@ -12626,6 +12655,7 @@ fn analyzeSwitchRuntimeBlock( item_ref, info.has_tag_capture, ); + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -12656,8 +12686,8 @@ fn analyzeSwitchRuntimeBlock( } })); emit_bb = true; - if (analyze_body) { - try spa.analyzeProngRuntime( + const prong_hint: std.builtin.BranchHint = if (analyze_body) h: { + break :h try spa.analyzeProngRuntime( &case_block, .normal, body, @@ -12670,9 +12700,11 @@ fn analyzeSwitchRuntimeBlock( item, info.has_tag_capture, ); - } else { + } else h: { _ = try case_block.addNoOp(.unreach); - } + break :h .none; + }; + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -12704,10 +12736,13 @@ fn analyzeSwitchRuntimeBlock( const body = sema.code.bodySlice(extra_index, info.body_len); extra_index += info.body_len; - if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) { - // nothing to do here - } else if (analyze_body) { - try spa.analyzeProngRuntime( + const prong_hint: std.builtin.BranchHint = if (err_set and + try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) + h: { + // nothing to do here. weight against error branch + break :h .unlikely; + } else if (analyze_body) h: { + break :h try spa.analyzeProngRuntime( &case_block, .normal, body, @@ -12720,10 +12755,12 @@ fn analyzeSwitchRuntimeBlock( .none, false, ); - } else { + } else h: { _ = try case_block.addNoOp(.unreach); - } + break :h .none; + }; + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 2 + items.len + case_block.instructions.items.len); @@ -12791,23 +12828,24 @@ fn analyzeSwitchRuntimeBlock( const body = sema.code.bodySlice(extra_index, info.body_len); extra_index += info.body_len; - if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) { - // nothing to do here - } else { - try spa.analyzeProngRuntime( - &case_block, - .normal, - body, - info.capture, - child_block.src(.{ .switch_capture = .{ - .switch_node_offset = switch_node_offset, - .case_idx = .{ .kind = .multi, .index = @intCast(multi_i) }, - } }), - items, - .none, - false, - ); - } + const prong_hint: std.builtin.BranchHint = if (err_set and + try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) + h: { + // nothing to do here. weight against error branch + break :h .unlikely; + } else try spa.analyzeProngRuntime( + &case_block, + .normal, + body, + info.capture, + child_block.src(.{ .switch_capture = .{ + .switch_node_offset = switch_node_offset, + .case_idx = .{ .kind = .multi, .index = @intCast(multi_i) }, + } }), + items, + .none, + false, + ); if (is_first) { is_first = false; @@ -12819,10 +12857,10 @@ fn analyzeSwitchRuntimeBlock( @typeInfo(Air.CondBr).Struct.fields.len + prev_then_body.len + cond_body.len, ); - sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = - sema.addExtraAssumeCapacity(Air.CondBr{ + sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = sema.addExtraAssumeCapacity(Air.CondBr{ .then_body_len = @intCast(prev_then_body.len), .else_body_len = @intCast(cond_body.len), + .branch_hints = .{ .true = prev_hint, .false = .none }, }); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(prev_then_body)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(cond_body)); @@ -12830,6 +12868,7 @@ fn analyzeSwitchRuntimeBlock( gpa.free(prev_then_body); prev_then_body = try case_block.instructions.toOwnedSlice(gpa); prev_cond_br = new_cond_br; + prev_hint = prong_hint; } } @@ -12861,8 +12900,8 @@ fn analyzeSwitchRuntimeBlock( if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src); emit_bb = true; - if (analyze_body) { - try spa.analyzeProngRuntime( + const prong_hint: std.builtin.BranchHint = if (analyze_body) h: { + break :h try spa.analyzeProngRuntime( &case_block, .special, special.body, @@ -12875,9 +12914,11 @@ fn analyzeSwitchRuntimeBlock( item_ref, special.has_tag_capture, ); - } else { + } else h: { _ = try case_block.addNoOp(.unreach); - } + break :h .none; + }; + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -12910,7 +12951,7 @@ fn analyzeSwitchRuntimeBlock( if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src); emit_bb = true; - try spa.analyzeProngRuntime( + const prong_hint = try spa.analyzeProngRuntime( &case_block, .special, special.body, @@ -12923,6 +12964,7 @@ fn analyzeSwitchRuntimeBlock( item_ref, special.has_tag_capture, ); + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -12944,7 +12986,7 @@ fn analyzeSwitchRuntimeBlock( if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src); emit_bb = true; - try spa.analyzeProngRuntime( + const prong_hint = try spa.analyzeProngRuntime( &case_block, .special, special.body, @@ -12957,6 +12999,7 @@ fn analyzeSwitchRuntimeBlock( item_ref, special.has_tag_capture, ); + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -12975,7 +13018,7 @@ fn analyzeSwitchRuntimeBlock( if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src); emit_bb = true; - try spa.analyzeProngRuntime( + const prong_hint = try spa.analyzeProngRuntime( &case_block, .special, special.body, @@ -12988,6 +13031,7 @@ fn analyzeSwitchRuntimeBlock( .bool_true, special.has_tag_capture, ); + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -13004,7 +13048,7 @@ fn analyzeSwitchRuntimeBlock( if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src); emit_bb = true; - try spa.analyzeProngRuntime( + const prong_hint = try spa.analyzeProngRuntime( &case_block, .special, special.body, @@ -13017,6 +13061,7 @@ fn analyzeSwitchRuntimeBlock( .bool_false, special.has_tag_capture, ); + try branch_hints.append(gpa, prong_hint); try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len); cases_extra.appendAssumeCapacity(1); // items_len @@ -13052,12 +13097,13 @@ fn analyzeSwitchRuntimeBlock( } else false else true; - if (special.body.len != 0 and err_set and + const else_hint: std.builtin.BranchHint = if (special.body.len != 0 and err_set and try sema.maybeErrorUnwrap(&case_block, special.body, operand, operand_src, allow_err_code_unwrap)) - { - // nothing to do here - } else if (special.body.len != 0 and analyze_body and !special.is_inline) { - try spa.analyzeProngRuntime( + h: { + // nothing to do here. weight against error branch + break :h .unlikely; + } else if (special.body.len != 0 and analyze_body and !special.is_inline) h: { + break :h try spa.analyzeProngRuntime( &case_block, .special, special.body, @@ -13070,7 +13116,7 @@ fn analyzeSwitchRuntimeBlock( .none, false, ); - } else { + } else h: { // We still need a terminator in this block, but we have proven // that it is unreachable. if (case_block.wantSafety()) { @@ -13079,33 +13125,57 @@ fn analyzeSwitchRuntimeBlock( } else { _ = try case_block.addNoOp(.unreach); } - } + // Safety check / unreachable branches are cold. + break :h .cold; + }; if (is_first) { + try branch_hints.append(gpa, else_hint); final_else_body = case_block.instructions.items; } else { + try branch_hints.append(gpa, .none); // we have the range conditionals first try sema.air_extra.ensureUnusedCapacity(gpa, prev_then_body.len + @typeInfo(Air.CondBr).Struct.fields.len + case_block.instructions.items.len); - sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = - sema.addExtraAssumeCapacity(Air.CondBr{ + sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = sema.addExtraAssumeCapacity(Air.CondBr{ .then_body_len = @intCast(prev_then_body.len), .else_body_len = @intCast(case_block.instructions.items.len), + .branch_hints = .{ .true = prev_hint, .false = else_hint }, }); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(prev_then_body)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(case_block.instructions.items)); final_else_body = first_else_body; } + } else { + try branch_hints.append(gpa, .none); } + assert(branch_hints.items.len == cases_len + 1); + try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).Struct.fields.len + - cases_extra.items.len + final_else_body.len); + cases_extra.items.len + final_else_body.len + + (std.math.divCeil(usize, branch_hints.items.len, 10) catch unreachable)); // branch hints const payload_index = sema.addExtraAssumeCapacity(Air.SwitchBr{ .cases_len = @intCast(cases_len), .else_body_len = @intCast(final_else_body.len), }); + { + // Add branch hints. + var cur_bag: u32 = 0; + for (branch_hints.items, 0..) |hint, idx| { + const idx_in_bag = idx % 10; + cur_bag |= @as(u32, @intFromEnum(hint)) << @intCast(idx_in_bag * 3); + if (idx_in_bag == 9) { + sema.air_extra.appendAssumeCapacity(cur_bag); + cur_bag = 0; + } + } + if (branch_hints.items.len % 10 != 0) { + sema.air_extra.appendAssumeCapacity(cur_bag); + } + } sema.air_extra.appendSliceAssumeCapacity(@ptrCast(cases_extra.items)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(final_else_body)); @@ -19167,6 +19237,10 @@ fn zirBoolBr( const lhs_result: Air.Inst.Ref = if (is_bool_or) .bool_true else .bool_false; _ = try lhs_block.addBr(block_inst, lhs_result); + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint; + sema.branch_hint = null; + const rhs_result = try sema.resolveInlineBody(rhs_block, body, inst); const rhs_noret = sema.typeOf(rhs_result).isNoReturn(mod); const coerced_rhs_result = if (!rhs_noret) rhs: { @@ -19175,7 +19249,17 @@ fn zirBoolBr( break :rhs coerced_result; } else rhs_result; - const result = sema.finishCondBr(parent_block, &child_block, &then_block, &else_block, lhs, block_inst); + const rhs_hint = sema.branch_hint orelse .none; + + const result = try sema.finishCondBr( + parent_block, + &child_block, + &then_block, + &else_block, + lhs, + block_inst, + if (is_bool_or) .{ .true = .none, .false = rhs_hint } else .{ .true = rhs_hint, .false = .none }, + ); if (!rhs_noret) { if (try sema.resolveDefinedValue(rhs_block, rhs_src, coerced_rhs_result)) |rhs_val| { if (is_bool_or and rhs_val.toBool()) { @@ -19197,6 +19281,7 @@ fn finishCondBr( else_block: *Block, cond: Air.Inst.Ref, block_inst: Air.Inst.Index, + branch_hints: Air.CondBr.BranchHints, ) !Air.Inst.Ref { const gpa = sema.gpa; @@ -19207,6 +19292,7 @@ fn finishCondBr( const cond_br_payload = sema.addExtraAssumeCapacity(Air.CondBr{ .then_body_len = @intCast(then_block.instructions.items.len), .else_body_len = @intCast(else_block.instructions.items.len), + .branch_hints = branch_hints, }); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items)); @@ -19341,6 +19427,11 @@ fn zirCondbr( if (try sema.resolveDefinedValue(parent_block, cond_src, cond)) |cond_val| { const body = if (cond_val.toBool()) then_body else else_body; + // We can propagate `.cold` hints from this branch since it's comptime-known + // to be taken from the parent branch. + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null; + try sema.maybeErrorUnwrapCondbr(parent_block, body, extra.data.condition, cond_src); // We use `analyzeBodyInner` since we want to propagate any comptime control flow to the caller. return sema.analyzeBodyInner(parent_block, body); @@ -19357,7 +19448,7 @@ fn zirCondbr( sub_block.need_debug_scope = null; // this body is emitted regardless defer sub_block.instructions.deinit(gpa); - try sema.analyzeBodyRuntimeBreak(&sub_block, then_body); + const true_hint = try sema.analyzeBodyRuntimeBreak(&sub_block, then_body); const true_instructions = try sub_block.instructions.toOwnedSlice(gpa); defer gpa.free(true_instructions); @@ -19373,11 +19464,13 @@ fn zirCondbr( break :blk try sub_block.addTyOp(.unwrap_errunion_err, result_ty, err_operand); }; - if (err_cond != null and try sema.maybeErrorUnwrap(&sub_block, else_body, err_cond.?, cond_src, false)) { - // nothing to do - } else { - try sema.analyzeBodyRuntimeBreak(&sub_block, else_body); - } + const false_hint: std.builtin.BranchHint = if (err_cond != null and + try sema.maybeErrorUnwrap(&sub_block, else_body, err_cond.?, cond_src, false)) + h: { + // nothing to do here. weight against error branch + break :h .unlikely; + } else try sema.analyzeBodyRuntimeBreak(&sub_block, else_body); + try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.CondBr).Struct.fields.len + true_instructions.len + sub_block.instructions.items.len); _ = try parent_block.addInst(.{ @@ -19387,6 +19480,7 @@ fn zirCondbr( .payload = sema.addExtraAssumeCapacity(Air.CondBr{ .then_body_len = @intCast(true_instructions.len), .else_body_len = @intCast(sub_block.instructions.items.len), + .branch_hints = .{ .true = true_hint, .false = false_hint }, }), } }, }); @@ -19411,6 +19505,11 @@ fn zirTry(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileError! } const is_non_err = try sema.analyzeIsNonErrComptimeOnly(parent_block, operand_src, err_union); if (is_non_err != .none) { + // We can propagate `.cold` hints from this branch since it's comptime-known + // to be taken from the parent branch. + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null; + const is_non_err_val = (try sema.resolveDefinedValue(parent_block, operand_src, is_non_err)).?; if (is_non_err_val.toBool()) { return sema.analyzeErrUnionPayload(parent_block, src, err_union_ty, err_union, operand_src, false); @@ -19424,13 +19523,19 @@ fn zirTry(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileError! var sub_block = parent_block.makeSubBlock(); defer sub_block.instructions.deinit(sema.gpa); + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint; + // This body is guaranteed to end with noreturn and has no breaks. try sema.analyzeBodyInner(&sub_block, body); + // The only interesting hint here is `.cold`, which can come from e.g. `errdefer @panic`. + const is_cold = sema.branch_hint == .cold; + try sema.air_extra.ensureUnusedCapacity(sema.gpa, @typeInfo(Air.Try).Struct.fields.len + sub_block.instructions.items.len); const try_inst = try parent_block.addInst(.{ - .tag = .@"try", + .tag = if (is_cold) .try_cold else .@"try", .data = .{ .pl_op = .{ .operand = err_union, .payload = sema.addExtraAssumeCapacity(Air.Try{ @@ -19460,6 +19565,11 @@ fn zirTryPtr(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileErr } const is_non_err = try sema.analyzeIsNonErrComptimeOnly(parent_block, operand_src, err_union); if (is_non_err != .none) { + // We can propagate `.cold` hints from this branch since it's comptime-known + // to be taken from the parent branch. + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null; + const is_non_err_val = (try sema.resolveDefinedValue(parent_block, operand_src, is_non_err)).?; if (is_non_err_val.toBool()) { return sema.analyzeErrUnionPayloadPtr(parent_block, src, operand, false, false); @@ -19473,9 +19583,15 @@ fn zirTryPtr(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileErr var sub_block = parent_block.makeSubBlock(); defer sub_block.instructions.deinit(sema.gpa); + const parent_hint = sema.branch_hint; + defer sema.branch_hint = parent_hint; + // This body is guaranteed to end with noreturn and has no breaks. try sema.analyzeBodyInner(&sub_block, body); + // The only interesting hint here is `.cold`, which can come from e.g. `errdefer @panic`. + const is_cold = sema.branch_hint == .cold; + const operand_ty = sema.typeOf(operand); const ptr_info = operand_ty.ptrInfo(mod); const res_ty = try pt.ptrTypeSema(.{ @@ -19491,7 +19607,7 @@ fn zirTryPtr(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileErr try sema.air_extra.ensureUnusedCapacity(sema.gpa, @typeInfo(Air.TryPtr).Struct.fields.len + sub_block.instructions.items.len); const try_inst = try parent_block.addInst(.{ - .tag = .try_ptr, + .tag = if (is_cold) .try_ptr_cold else .try_ptr, .data = .{ .ty_pl = .{ .ty = res_ty_ref, .payload = sema.addExtraAssumeCapacity(Air.TryPtr{ @@ -19743,6 +19859,8 @@ fn retWithErrTracing( const cond_br_payload = sema.addExtraAssumeCapacity(Air.CondBr{ .then_body_len = @intCast(then_block.instructions.items.len), .else_body_len = @intCast(else_block.instructions.items.len), + // weight against error branch + .branch_hints = .{ .true = .likely, .false = .unlikely }, }); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items)); @@ -26749,6 +26867,7 @@ fn zirBuiltinValue(sema: *Sema, extended: Zir.Inst.Extended.InstData) CompileErr .export_options => "ExportOptions", .extern_options => "ExternOptions", .type_info => "Type", + .branch_hint => "BranchHint", // Values are handled here. .calling_convention_c => { @@ -26774,6 +26893,27 @@ fn zirBuiltinValue(sema: *Sema, extended: Zir.Inst.Extended.InstData) CompileErr return Air.internedToRef(ty.toIntern()); } +fn zirBranchHint(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!void { + const pt = sema.pt; + const zcu = pt.zcu; + + const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data; + const uncoerced_hint = try sema.resolveInst(extra.operand); + const operand_src = block.builtinCallArgSrc(extra.node, 0); + + const hint_ty = try pt.getBuiltinType("BranchHint"); + const coerced_hint = try sema.coerce(block, hint_ty, uncoerced_hint, operand_src); + const hint_val = try sema.resolveConstDefinedValue(block, operand_src, coerced_hint, .{ + .needed_comptime_reason = "operand to '@branchHint' must be comptime-known", + }); + + // We only apply the first hint in a branch. + // This allows user-provided hints to override implicit cold hints. + if (sema.branch_hint == null) { + sema.branch_hint = zcu.toEnum(std.builtin.BranchHint, hint_val); + } +} + fn requireRuntimeBlock(sema: *Sema, block: *Block, src: LazySrcLoc, runtime_src: ?LazySrcLoc) !void { if (block.is_comptime) { const msg = msg: { @@ -27329,13 +27469,17 @@ fn addSafetyCheckExtra( sema.air_instructions.appendAssumeCapacity(.{ .tag = .cond_br, - .data = .{ .pl_op = .{ - .operand = ok, - .payload = sema.addExtraAssumeCapacity(Air.CondBr{ - .then_body_len = 1, - .else_body_len = @intCast(fail_block.instructions.items.len), - }), - } }, + .data = .{ + .pl_op = .{ + .operand = ok, + .payload = sema.addExtraAssumeCapacity(Air.CondBr{ + .then_body_len = 1, + .else_body_len = @intCast(fail_block.instructions.items.len), + // safety check failure branch is cold + .branch_hints = .{ .true = .likely, .false = .cold }, + }), + }, + }, }); sema.air_extra.appendAssumeCapacity(@intFromEnum(br_inst)); sema.air_extra.appendSliceAssumeCapacity(@ptrCast(fail_block.instructions.items)); @@ -27532,6 +27676,7 @@ fn safetyCheckFormatted( try sema.addSafetyCheckExtra(parent_block, ok, &fail_block); } +/// This does not set `sema.branch_hint`. fn safetyPanic(sema: *Sema, block: *Block, src: LazySrcLoc, panic_id: Module.PanicId) CompileError!void { const msg_nav_index = try sema.preparePanicId(block, src, panic_id); const msg_inst = try sema.analyzeNavVal(block, src, msg_nav_index); @@ -37244,7 +37389,7 @@ pub fn addExtraAssumeCapacity(sema: *Sema, extra: anytype) u32 { inline for (fields) |field| { sema.air_extra.appendAssumeCapacity(switch (field.type) { u32 => @field(extra, field.name), - i32 => @bitCast(@field(extra, field.name)), + i32, Air.CondBr.BranchHints => @bitCast(@field(extra, field.name)), Air.Inst.Ref, InternPool.Index => @intFromEnum(@field(extra, field.name)), else => @compileError("bad field type: " ++ @typeName(field.type)), }); @@ -38338,6 +38483,12 @@ fn maybeDerefSliceAsArray( fn analyzeUnreachable(sema: *Sema, block: *Block, src: LazySrcLoc, safety_check: bool) !void { if (safety_check and block.wantSafety()) { + // We only apply the first hint in a branch. + // This allows user-provided hints to override implicit cold hints. + if (sema.branch_hint == null) { + sema.branch_hint = .cold; + } + try sema.safetyPanic(block, src, .unreach); } else { _ = try block.addNoOp(.unreach); diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 01063ab2ce0e..427366367545 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -2188,6 +2188,8 @@ fn analyzeFnBody(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaError! }); } + func.setBranchHint(ip, sema.branch_hint orelse .none); + // If we don't get an error return trace from a caller, create our own. if (func.analysisUnordered(ip).calls_or_awaits_errorable_fn and zcu.comp.config.any_error_tracing and diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 882f3e98e3e1..62cea5521e3c 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -795,7 +795,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}), .@"try" => try self.airTry(inst), + .try_cold => try self.airTry(inst), .try_ptr => try self.airTryPtr(inst), + .try_ptr_cold => try self.airTryPtr(inst), .dbg_stmt => try self.airDbgStmt(inst), .dbg_inline_block => try self.airDbgInlineBlock(inst), @@ -5093,25 +5095,17 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! } fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition_ty = self.typeOf(pl_op.operand); - const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload); + const switch_br = self.air.unwrapSwitch(inst); + const condition_ty = self.typeOf(switch_br.operand); const liveness = try self.liveness.getSwitchBr( self.gpa, inst, - switch_br.data.cases_len + 1, + switch_br.cases_len + 1, ); defer self.gpa.free(liveness.deaths); - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[case.end..][0..case.data.items_len])); - assert(items.len > 0); - const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + items.len + case_body.len; - + var it = switch_br.iterateCases(); + while (it.next()) |case| { // For every item, we compare it to condition and branch into // the prong if they are equal. After we compared to all // items, we branch into the next prong (or if no other prongs @@ -5127,11 +5121,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { // prong: ... // ... // out: ... - const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len); + const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len); defer self.gpa.free(branch_into_prong_relocs); - for (items, 0..) |item, idx| { - const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq); + for (case.items, 0..) |item, idx| { + const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq); branch_into_prong_relocs[idx] = try self.condBr(cmp_result); } @@ -5157,11 +5151,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { _ = self.branch_stack.pop(); } - try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len); - for (liveness.deaths[case_i]) |operand| { + try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len); + for (liveness.deaths[case.idx]) |operand| { self.processDeath(operand); } - try self.genBody(case_body); + try self.genBody(case.body); // Revert to the previous register and stack allocation state. var saved_case_branch = self.branch_stack.pop(); @@ -5179,8 +5173,8 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { try self.performReloc(branch_away_from_prong_reloc); } - if (switch_br.data.else_body_len > 0) { - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]); + if (switch_br.else_body_len > 0) { + const else_body = it.elseBody(); // Capture the state of register and stack allocation state so that we can revert to it. const parent_next_stack_offset = self.next_stack_offset; @@ -5219,7 +5213,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { // in airCondBr. } - return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none }); + return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none }); } fn performReloc(self: *Self, inst: Mir.Inst.Index) !void { @@ -6362,14 +6356,14 @@ fn wantSafety(self: *Self) bool { } fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); self.err_msg = try ErrorMsg.create(self.gpa, self.src_loc, format, args); return error.CodegenFail; } fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); self.err_msg = try ErrorMsg.create(self.gpa, self.src_loc, format, args); return error.CodegenFail; diff --git a/src/arch/aarch64/Emit.zig b/src/arch/aarch64/Emit.zig index 641a6fd920a3..6c5630718365 100644 --- a/src/arch/aarch64/Emit.zig +++ b/src/arch/aarch64/Emit.zig @@ -430,7 +430,7 @@ fn writeInstruction(emit: *Emit, instruction: Instruction) !void { } fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(emit.err_msg == null); const comp = emit.bin_file.comp; const gpa = comp.gpa; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index 796d3e34dc94..827cc606b531 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -782,7 +782,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}), .@"try" => try self.airTry(inst), + .try_cold => try self.airTry(inst), .try_ptr => try self.airTryPtr(inst), + .try_ptr_cold => try self.airTryPtr(inst), .dbg_stmt => try self.airDbgStmt(inst), .dbg_inline_block => try self.airDbgInlineBlock(inst), @@ -5040,25 +5042,17 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! } fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition_ty = self.typeOf(pl_op.operand); - const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload); + const switch_br = self.air.unwrapSwitch(inst); + const condition_ty = self.typeOf(switch_br.operand); const liveness = try self.liveness.getSwitchBr( self.gpa, inst, - switch_br.data.cases_len + 1, + switch_br.cases_len + 1, ); defer self.gpa.free(liveness.deaths); - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]); - assert(items.len > 0); - const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + items.len + case_body.len; - + var it = switch_br.iterateCases(); + while (it.next()) |case| { // For every item, we compare it to condition and branch into // the prong if they are equal. After we compared to all // items, we branch into the next prong (or if no other prongs @@ -5074,11 +5068,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { // prong: ... // ... // out: ... - const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len); + const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len); defer self.gpa.free(branch_into_prong_relocs); - for (items, 0..) |item, idx| { - const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq); + for (case.items, 0..) |item, idx| { + const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq); branch_into_prong_relocs[idx] = try self.condBr(cmp_result); } @@ -5104,11 +5098,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { _ = self.branch_stack.pop(); } - try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len); - for (liveness.deaths[case_i]) |operand| { + try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len); + for (liveness.deaths[case.idx]) |operand| { self.processDeath(operand); } - try self.genBody(case_body); + try self.genBody(case.body); // Revert to the previous register and stack allocation state. var saved_case_branch = self.branch_stack.pop(); @@ -5126,8 +5120,8 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { try self.performReloc(branch_away_from_prong_reloc); } - if (switch_br.data.else_body_len > 0) { - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]); + if (switch_br.else_body_len > 0) { + const else_body = it.elseBody(); // Capture the state of register and stack allocation state so that we can revert to it. const parent_next_stack_offset = self.next_stack_offset; @@ -5166,7 +5160,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { // in airCondBr. } - return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none }); + return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none }); } fn performReloc(self: *Self, inst: Mir.Inst.Index) !void { @@ -6319,7 +6313,7 @@ fn wantSafety(self: *Self) bool { } fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); const gpa = self.gpa; self.err_msg = try ErrorMsg.create(gpa, self.src_loc, format, args); @@ -6327,7 +6321,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError { } fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); const gpa = self.gpa; self.err_msg = try ErrorMsg.create(gpa, self.src_loc, format, args); diff --git a/src/arch/arm/Emit.zig b/src/arch/arm/Emit.zig index dd68f2397da5..c1b5baad578f 100644 --- a/src/arch/arm/Emit.zig +++ b/src/arch/arm/Emit.zig @@ -348,7 +348,7 @@ fn writeInstruction(emit: *Emit, instruction: Instruction) !void { } fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(emit.err_msg == null); const comp = emit.bin_file.comp; const gpa = comp.gpa; diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index deeb0dc4da88..2320d0b4dce6 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -1640,7 +1640,9 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { .addrspace_cast => return func.fail("TODO: addrspace_cast", .{}), .@"try" => try func.airTry(inst), + .try_cold => try func.airTry(inst), .try_ptr => return func.fail("TODO: try_ptr", .{}), + .try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}), .dbg_var_ptr, .dbg_var_val, @@ -5655,38 +5657,30 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! } fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition_ty = func.typeOf(pl_op.operand); - const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload); - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1); + const switch_br = func.air.unwrapSwitch(inst); + + const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1); defer func.gpa.free(liveness.deaths); - const condition = try func.resolveInst(pl_op.operand); + const condition = try func.resolveInst(switch_br.operand); + const condition_ty = func.typeOf(switch_br.operand); // If the condition dies here in this switch instruction, process // that death now instead of later as this has an effect on // whether it needs to be spilled in the branches if (func.liveness.operandDies(inst, 0)) { - if (pl_op.operand.toIndex()) |op_inst| try func.processDeath(op_inst); + if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst); } func.scope_generation += 1; const state = try func.saveState(); - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = func.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = - @ptrCast(func.air.extra[case.end..][0..case.data.items_len]); - const case_body: []const Air.Inst.Index = - @ptrCast(func.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + items.len + case_body.len; - - var relocs = try func.gpa.alloc(Mir.Inst.Index, items.len); + var it = switch_br.iterateCases(); + while (it.next()) |case| { + var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len); defer func.gpa.free(relocs); - for (items, relocs, 0..) |item, *reloc, i| { + for (case.items, relocs, 0..) |item, *reloc, i| { const item_mcv = try func.resolveInst(item); const cond_lock = switch (condition) { @@ -5720,10 +5714,10 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg }); } - for (liveness.deaths[case_i]) |operand| try func.processDeath(operand); + for (liveness.deaths[case.idx]) |operand| try func.processDeath(operand); for (relocs[0 .. relocs.len - 1]) |reloc| func.performReloc(reloc); - try func.genBody(case_body); + try func.genBody(case.body); try func.restoreState(state, &.{}, .{ .emit_instructions = false, .update_tracking = true, @@ -5734,9 +5728,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void { func.performReloc(relocs[relocs.len - 1]); } - if (switch_br.data.else_body_len > 0) { - const else_body: []const Air.Inst.Index = - @ptrCast(func.air.extra[extra_index..][0..switch_br.data.else_body_len]); + if (switch_br.else_body_len > 0) { + const else_body = it.elseBody(); const else_deaths = liveness.deaths.len - 1; for (liveness.deaths[else_deaths]) |operand| try func.processDeath(operand); @@ -8222,14 +8215,14 @@ fn wantSafety(func: *Func) bool { } fn fail(func: *Func, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(func.err_msg == null); func.err_msg = try ErrorMsg.create(func.gpa, func.src_loc, format, args); return error.CodegenFail; } fn failSymbol(func: *Func, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(func.err_msg == null); func.err_msg = try ErrorMsg.create(func.gpa, func.src_loc, format, args); return error.CodegenFail; diff --git a/src/arch/riscv64/Lower.zig b/src/arch/riscv64/Lower.zig index fc8202adeafd..cbcce9f21579 100644 --- a/src/arch/riscv64/Lower.zig +++ b/src/arch/riscv64/Lower.zig @@ -582,7 +582,7 @@ fn pushPopRegList(lower: *Lower, comptime spilling: bool, reg_list: Mir.Register } pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { - @setCold(true); + @branchHint(.cold); assert(lower.err_msg == null); lower.err_msg = try ErrorMsg.create(lower.allocator, lower.src_loc, format, args); return error.LowerFail; diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index 99192aa55448..73c99fc86864 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -637,7 +637,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .addrspace_cast => @panic("TODO try self.airAddrSpaceCast(int)"), .@"try" => try self.airTry(inst), + .try_cold => try self.airTry(inst), .try_ptr => @panic("TODO try self.airTryPtr(inst)"), + .try_ptr_cold => @panic("TODO try self.airTryPtrCold(inst)"), .dbg_stmt => try self.airDbgStmt(inst), .dbg_inline_block => try self.airDbgInlineBlock(inst), @@ -3533,7 +3535,7 @@ fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) } fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); const gpa = self.gpa; self.err_msg = try ErrorMsg.create(gpa, self.src_loc, format, args); diff --git a/src/arch/sparc64/Emit.zig b/src/arch/sparc64/Emit.zig index 165ee1c45e78..4e49bcf5b42e 100644 --- a/src/arch/sparc64/Emit.zig +++ b/src/arch/sparc64/Emit.zig @@ -511,7 +511,7 @@ fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) !void { } fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(emit.err_msg == null); const comp = emit.bin_file.comp; const gpa = comp.gpa; diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index d2e9db806287..546fe51b3fbb 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -1913,7 +1913,9 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .get_union_tag => func.airGetUnionTag(inst), .@"try" => func.airTry(inst), + .try_cold => func.airTry(inst), .try_ptr => func.airTryPtr(inst), + .try_ptr_cold => func.airTryPtr(inst), .dbg_stmt => func.airDbgStmt(inst), .dbg_inline_block => func.airDbgInlineBlock(inst), @@ -4039,37 +4041,31 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const mod = pt.zcu; // result type is always 'noreturn' const blocktype = wasm.block_empty; - const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const target = try func.resolveInst(pl_op.operand); - const target_ty = func.typeOf(pl_op.operand); - const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload); - const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1); + const switch_br = func.air.unwrapSwitch(inst); + const target = try func.resolveInst(switch_br.operand); + const target_ty = func.typeOf(switch_br.operand); + const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1); defer func.gpa.free(liveness.deaths); - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - // a list that maps each value with its value and body based on the order inside the list. const CaseValue = struct { integer: i32, value: Value }; var case_list = try std.ArrayList(struct { values: []const CaseValue, body: []const Air.Inst.Index, - }).initCapacity(func.gpa, switch_br.data.cases_len); + }).initCapacity(func.gpa, switch_br.cases_len); defer for (case_list.items) |case| { func.gpa.free(case.values); } else case_list.deinit(); var lowest_maybe: ?i32 = null; var highest_maybe: ?i32 = null; - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = func.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = @ptrCast(func.air.extra[case.end..][0..case.data.items_len]); - const case_body: []const Air.Inst.Index = @ptrCast(func.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + items.len + case_body.len; - const values = try func.gpa.alloc(CaseValue, items.len); + + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const values = try func.gpa.alloc(CaseValue, case.items.len); errdefer func.gpa.free(values); - for (items, 0..) |ref, i| { + for (case.items, 0..) |ref, i| { const item_val = (try func.air.value(ref, pt)).?; const int_val = func.valueAsI32(item_val); if (lowest_maybe == null or int_val < lowest_maybe.?) { @@ -4081,7 +4077,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { values[i] = .{ .integer = int_val, .value = item_val }; } - case_list.appendAssumeCapacity(.{ .values = values, .body = case_body }); + case_list.appendAssumeCapacity(.{ .values = values, .body = case.body }); try func.startBlock(.block, blocktype); } @@ -4095,7 +4091,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { // TODO: Benchmark this to find a proper value, LLVM seems to draw the line at '40~45'. const is_sparse = highest - lowest > 50 or target_ty.bitSize(pt) > 32; - const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra_index..][0..switch_br.data.else_body_len]); + const else_body = it.elseBody(); const has_else_body = else_body.len != 0; if (has_else_body) { try func.startBlock(.block, blocktype); @@ -4138,11 +4134,11 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { // for errors that are not present in any branch. This is fine as this default // case will never be hit for those cases but we do save runtime cost and size // by using a jump table for this instead of if-else chains. - break :blk if (has_else_body or target_ty.zigTypeTag(mod) == .ErrorSet) case_i else unreachable; + break :blk if (has_else_body or target_ty.zigTypeTag(mod) == .ErrorSet) switch_br.cases_len else unreachable; }; func.mir_extra.appendAssumeCapacity(idx); } else if (has_else_body) { - func.mir_extra.appendAssumeCapacity(case_i); // default branch + func.mir_extra.appendAssumeCapacity(switch_br.cases_len); // default branch } try func.endBlock(); } diff --git a/src/arch/wasm/Emit.zig b/src/arch/wasm/Emit.zig index d795e2afa681..aa20b0ad3551 100644 --- a/src/arch/wasm/Emit.zig +++ b/src/arch/wasm/Emit.zig @@ -252,7 +252,7 @@ fn offset(self: Emit) u32 { } fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); std.debug.assert(emit.error_msg == null); const comp = emit.bin_file.base.comp; const zcu = comp.module.?; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 316389a7c25f..576e8e760e40 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2262,7 +2262,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}), .@"try" => try self.airTry(inst), + .try_cold => try self.airTry(inst), // TODO .try_ptr => try self.airTryPtr(inst), + .try_ptr_cold => try self.airTryPtr(inst), // TODO .dbg_stmt => try self.airDbgStmt(inst), .dbg_inline_block => try self.airDbgInlineBlock(inst), @@ -13629,38 +13631,29 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! } fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition = try self.resolveInst(pl_op.operand); - const condition_ty = self.typeOf(pl_op.operand); - const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload); - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.data.cases_len + 1); + const switch_br = self.air.unwrapSwitch(inst); + const condition = try self.resolveInst(switch_br.operand); + const condition_ty = self.typeOf(switch_br.operand); + const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1); defer self.gpa.free(liveness.deaths); // If the condition dies here in this switch instruction, process // that death now instead of later as this has an effect on // whether it needs to be spilled in the branches if (self.liveness.operandDies(inst, 0)) { - if (pl_op.operand.toIndex()) |op_inst| try self.processDeath(op_inst); + if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); } self.scope_generation += 1; const state = try self.saveState(); - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = - @ptrCast(self.air.extra[case.end..][0..case.data.items_len]); - const case_body: []const Air.Inst.Index = - @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + items.len + case_body.len; - - var relocs = try self.gpa.alloc(Mir.Inst.Index, items.len); + var it = switch_br.iterateCases(); + while (it.next()) |case| { + var relocs = try self.gpa.alloc(Mir.Inst.Index, case.items.len); defer self.gpa.free(relocs); try self.spillEflagsIfOccupied(); - for (items, relocs, 0..) |item, *reloc, i| { + for (case.items, relocs, 0..) |item, *reloc, i| { const item_mcv = try self.resolveInst(item); const cc: Condition = switch (condition) { .eflags => |cc| switch (item_mcv.immediate) { @@ -13676,10 +13669,10 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) cc else cc.negate(), undefined); } - for (liveness.deaths[case_i]) |operand| try self.processDeath(operand); + for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand); for (relocs[0 .. relocs.len - 1]) |reloc| self.performReloc(reloc); - try self.genBody(case_body); + try self.genBody(case.body); try self.restoreState(state, &.{}, .{ .emit_instructions = false, .update_tracking = true, @@ -13690,9 +13683,8 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { self.performReloc(relocs[relocs.len - 1]); } - if (switch_br.data.else_body_len > 0) { - const else_body: []const Air.Inst.Index = - @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]); + if (switch_br.else_body_len > 0) { + const else_body = it.elseBody(); const else_deaths = liveness.deaths.len - 1; for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand); @@ -19208,7 +19200,7 @@ fn resolveCallingConventionValues( } fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); const gpa = self.gpa; self.err_msg = try ErrorMsg.create(gpa, self.src_loc, format, args); @@ -19216,7 +19208,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError { } fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError { - @setCold(true); + @branchHint(.cold); assert(self.err_msg == null); const gpa = self.gpa; self.err_msg = try ErrorMsg.create(gpa, self.src_loc, format, args); diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 69cd548cc3e9..33ba0c7629db 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -293,7 +293,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { } pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error { - @setCold(true); + @branchHint(.cold); assert(lower.err_msg == null); lower.err_msg = try Zcu.ErrorMsg.create(lower.allocator, lower.src_loc, format, args); return error.LowerFail; diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 397cb071b640..f93cd01953af 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -626,7 +626,7 @@ pub const DeclGen = struct { } fn fail(dg: *DeclGen, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } { - @setCold(true); + @branchHint(.cold); const zcu = dg.pt.zcu; const src_loc = zcu.navSrcLoc(dg.pass.nav); dg.error_msg = try Zcu.ErrorMsg.create(dg.gpa, src_loc, format, args); @@ -1786,7 +1786,7 @@ pub const DeclGen = struct { else => unreachable, } } - if (fn_val.getFunction(zcu)) |func| if (func.analysisUnordered(ip).is_cold) + if (fn_val.getFunction(zcu)) |func| if (func.analysisUnordered(ip).branch_hint == .cold) try w.writeAll("zig_cold "); if (fn_info.return_type == .noreturn_type) try w.writeAll("zig_noreturn "); @@ -3288,8 +3288,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .prefetch => try airPrefetch(f, inst), .addrspace_cast => return f.fail("TODO: C backend: implement addrspace_cast", .{}), - .@"try" => try airTry(f, inst), - .try_ptr => try airTryPtr(f, inst), + .@"try" => try airTry(f, inst), + .try_cold => try airTry(f, inst), + .try_ptr => try airTryPtr(f, inst), + .try_ptr_cold => try airTryPtr(f, inst), .dbg_stmt => try airDbgStmt(f, inst), .dbg_inline_block => try airDbgInlineBlock(f, inst), @@ -4986,11 +4988,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { const pt = f.object.dg.pt; const zcu = pt.zcu; - const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const condition = try f.resolveInst(pl_op.operand); - try reap(f, inst, &.{pl_op.operand}); - const condition_ty = f.typeOf(pl_op.operand); - const switch_br = f.air.extraData(Air.SwitchBr, pl_op.payload); + const switch_br = f.air.unwrapSwitch(inst); + const condition = try f.resolveInst(switch_br.operand); + try reap(f, inst, &.{switch_br.operand}); + const condition_ty = f.typeOf(switch_br.operand); const writer = f.object.writer(); try writer.writeAll("switch ("); @@ -5011,22 +5012,16 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { f.object.indent_writer.pushIndent(); const gpa = f.object.dg.gpa; - const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.data.cases_len + 1); + const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.cases_len + 1); defer gpa.free(liveness.deaths); // On the final iteration we do not need to fix any state. This is because, like in the `else` // branch of a `cond_br`, our parent has to do it for this entire body anyway. - const last_case_i = switch_br.data.cases_len - @intFromBool(switch_br.data.else_body_len == 0); - - var extra_index: usize = switch_br.end; - for (0..switch_br.data.cases_len) |case_i| { - const case = f.air.extraData(Air.SwitchBr.Case, extra_index); - const items = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[case.end..][0..case.data.items_len])); - const case_body: []const Air.Inst.Index = - @ptrCast(f.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + case.data.items_len + case_body.len; + const last_case_i = switch_br.cases_len - @intFromBool(switch_br.else_body_len == 0); - for (items) |item| { + var it = switch_br.iterateCases(); + while (it.next()) |case| { + for (case.items) |item| { try f.object.indent_writer.insertNewline(); try writer.writeAll("case "); const item_value = try f.air.value(item, pt); @@ -5044,19 +5039,19 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { } try writer.writeByte(' '); - if (case_i != last_case_i) { - try genBodyResolveState(f, inst, liveness.deaths[case_i], case_body, false); + if (case.idx != last_case_i) { + try genBodyResolveState(f, inst, liveness.deaths[case.idx], case.body, false); } else { - for (liveness.deaths[case_i]) |death| { + for (liveness.deaths[case.idx]) |death| { try die(f, inst, death.toRef()); } - try genBody(f, case_body); + try genBody(f, case.body); } // The case body must be noreturn so we don't need to insert a break. } - const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra_index..][0..switch_br.data.else_body_len]); + const else_body = it.elseBody(); try f.object.indent_writer.insertNewline(); if (else_body.len > 0) { // Note that this must be the last case (i.e. the `last_case_i` case was not hit above) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 233cf7e3eba4..369cba3e8ebf 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -898,9 +898,9 @@ pub const Object = struct { const i32_2 = try builder.intConst(.i32, 2); const i32_3 = try builder.intConst(.i32, 3); const debug_info_version = try builder.debugModuleFlag( - try builder.debugConstant(i32_2), + try builder.metadataConstant(i32_2), try builder.metadataString("Debug Info Version"), - try builder.debugConstant(i32_3), + try builder.metadataConstant(i32_3), ); switch (comp.config.debug_format) { @@ -908,9 +908,9 @@ pub const Object = struct { .dwarf => |f| { const i32_4 = try builder.intConst(.i32, 4); const dwarf_version = try builder.debugModuleFlag( - try builder.debugConstant(i32_2), + try builder.metadataConstant(i32_2), try builder.metadataString("Dwarf Version"), - try builder.debugConstant(i32_4), + try builder.metadataConstant(i32_4), ); switch (f) { .@"32" => { @@ -921,9 +921,9 @@ pub const Object = struct { }, .@"64" => { const dwarf64 = try builder.debugModuleFlag( - try builder.debugConstant(i32_2), + try builder.metadataConstant(i32_2), try builder.metadataString("DWARF64"), - try builder.debugConstant(.@"1"), + try builder.metadataConstant(.@"1"), ); try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{ debug_info_version, @@ -935,9 +935,9 @@ pub const Object = struct { }, .code_view => { const code_view = try builder.debugModuleFlag( - try builder.debugConstant(i32_2), + try builder.metadataConstant(i32_2), try builder.metadataString("CodeView"), - try builder.debugConstant(.@"1"), + try builder.metadataConstant(.@"1"), ); try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{ debug_info_version, @@ -1122,12 +1122,12 @@ pub const Object = struct { self.builder.debugForwardReferenceSetType( self.debug_enums_fwd_ref, - try self.builder.debugTuple(self.debug_enums.items), + try self.builder.metadataTuple(self.debug_enums.items), ); self.builder.debugForwardReferenceSetType( self.debug_globals_fwd_ref, - try self.builder.debugTuple(self.debug_globals.items), + try self.builder.metadataTuple(self.debug_globals.items), ); } } @@ -1369,7 +1369,7 @@ pub const Object = struct { _ = try attributes.removeFnAttr(.alignstack); } - if (func_analysis.is_cold) { + if (func_analysis.branch_hint == .cold) { try attributes.addFnAttr(.cold, &o.builder); } else { _ = try attributes.removeFnAttr(.cold); @@ -1978,7 +1978,7 @@ pub const Object = struct { try o.lowerDebugType(int_ty), ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(enumerators), + try o.builder.metadataTuple(enumerators), ); try o.debug_type_map.put(gpa, ty, debug_enum_type); @@ -2087,7 +2087,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(&.{ + try o.builder.metadataTuple(&.{ debug_ptr_type, debug_len_type, }), @@ -2167,10 +2167,10 @@ pub const Object = struct { try o.lowerDebugType(ty.childType(zcu)), ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(&.{ + try o.builder.metadataTuple(&.{ try o.builder.debugSubrange( - try o.builder.debugConstant(try o.builder.intConst(.i64, 0)), - try o.builder.debugConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))), + try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)), + try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))), ), }), ); @@ -2210,10 +2210,10 @@ pub const Object = struct { debug_elem_type, ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(&.{ + try o.builder.metadataTuple(&.{ try o.builder.debugSubrange( - try o.builder.debugConstant(try o.builder.intConst(.i64, 0)), - try o.builder.debugConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))), + try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)), + try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))), ), }), ); @@ -2288,7 +2288,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(&.{ + try o.builder.metadataTuple(&.{ debug_data_type, debug_some_type, }), @@ -2367,7 +2367,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(&fields), + try o.builder.metadataTuple(&fields), ); o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_error_union_type); @@ -2447,7 +2447,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(fields.items), + try o.builder.metadataTuple(fields.items), ); o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type); @@ -2526,7 +2526,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(fields.items), + try o.builder.metadataTuple(fields.items), ); o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type); @@ -2567,7 +2567,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple( + try o.builder.metadataTuple( &.{try o.lowerDebugType(Type.fromInterned(union_type.enum_tag_ty))}, ), ); @@ -2629,7 +2629,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(fields.items), + try o.builder.metadataTuple(fields.items), ); o.builder.debugForwardReferenceSetType(debug_union_fwd_ref, debug_union_type); @@ -2688,7 +2688,7 @@ pub const Object = struct { .none, // Underlying type ty.abiSize(pt) * 8, (ty.abiAlignment(pt).toByteUnits() orelse 0) * 8, - try o.builder.debugTuple(&full_fields), + try o.builder.metadataTuple(&full_fields), ); o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_tagged_union_type); @@ -2741,7 +2741,7 @@ pub const Object = struct { } const debug_function_type = try o.builder.debugSubroutineType( - try o.builder.debugTuple(debug_param_types.items), + try o.builder.metadataTuple(debug_param_types.items), ); try o.debug_type_map.put(gpa, ty, debug_function_type); @@ -4585,7 +4585,7 @@ pub const Object = struct { const bad_value_block = try wip.block(1, "BadValue"); const tag_int_value = wip.arg(0); var wip_switch = - try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len)); + try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len), .none); defer wip_switch.finish(&wip); for (0..enum_type.names.len) |field_index| { @@ -4632,7 +4632,7 @@ pub const NavGen = struct { } fn todo(ng: *NavGen, comptime format: []const u8, args: anytype) Error { - @setCold(true); + @branchHint(.cold); assert(ng.err_msg == null); const o = ng.object; const gpa = o.gpa; @@ -4798,7 +4798,7 @@ pub const FuncGen = struct { } fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error { - @setCold(true); + @branchHint(.cold); return self.ng.todo(format, args); } @@ -4972,8 +4972,10 @@ pub const FuncGen = struct { .ret_addr => try self.airRetAddr(inst), .frame_addr => try self.airFrameAddress(inst), .cond_br => try self.airCondBr(inst), - .@"try" => try self.airTry(body[i..]), - .try_ptr => try self.airTryPtr(inst), + .@"try" => try self.airTry(body[i..], false), + .try_cold => try self.airTry(body[i..], true), + .try_ptr => try self.airTryPtr(inst, false), + .try_ptr_cold => try self.airTryPtr(inst, true), .intcast => try self.airIntCast(inst), .trunc => try self.airTrunc(inst), .fptrunc => try self.airFptrunc(inst), @@ -5520,6 +5522,7 @@ pub const FuncGen = struct { const panic_nav = ip.getNav(panic_func.owner_nav); const fn_info = zcu.typeToFunc(Type.fromInterned(panic_nav.typeOf(ip))).?; const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav); + _ = try fg.wip.callIntrinsicAssumeCold(); _ = try fg.wip.call( .normal, toLlvmCallConv(fn_info.cc, target), @@ -5806,7 +5809,7 @@ pub const FuncGen = struct { const mixed_block = try self.wip.block(1, "Mixed"); const both_pl_block = try self.wip.block(1, "BothNonNull"); const end_block = try self.wip.block(3, "End"); - var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2); + var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none); defer wip_switch.finish(&self.wip); try wip_switch.addCase( try o.builder.intConst(llvm_i2, 0b00), @@ -5960,21 +5963,62 @@ pub const FuncGen = struct { const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]); const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]); + const Hint = enum { + none, + unpredictable, + then_likely, + else_likely, + then_cold, + else_cold, + }; + const hint: Hint = switch (extra.data.branch_hints.true) { + .none => switch (extra.data.branch_hints.false) { + .none => .none, + .likely => .else_likely, + .unlikely => .then_likely, + .cold => .else_cold, + .unpredictable => .unpredictable, + }, + .likely => switch (extra.data.branch_hints.false) { + .none => .then_likely, + .likely => .unpredictable, + .unlikely => .then_likely, + .cold => .else_cold, + .unpredictable => .unpredictable, + }, + .unlikely => switch (extra.data.branch_hints.false) { + .none => .else_likely, + .likely => .else_likely, + .unlikely => .unpredictable, + .cold => .else_cold, + .unpredictable => .unpredictable, + }, + .cold => .then_cold, + .unpredictable => .unpredictable, + }; + const then_block = try self.wip.block(1, "Then"); const else_block = try self.wip.block(1, "Else"); - _ = try self.wip.brCond(cond, then_block, else_block); + _ = try self.wip.brCond(cond, then_block, else_block, switch (hint) { + .none, .then_cold, .else_cold => .none, + .unpredictable => .unpredictable, + .then_likely => .then_likely, + .else_likely => .else_likely, + }); self.wip.cursor = .{ .block = then_block }; + if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold(); try self.genBodyDebugScope(null, then_body); self.wip.cursor = .{ .block = else_block }; + if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold(); try self.genBodyDebugScope(null, else_body); // No need to reset the insert cursor since this instruction is noreturn. return .none; } - fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { + fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index, err_cold: bool) !Builder.Value { const o = self.ng.object; const pt = o.pt; const inst = body_tail[0]; @@ -5986,10 +6030,10 @@ pub const FuncGen = struct { const payload_ty = self.typeOfIndex(inst); const can_elide_load = if (isByRef(payload_ty, pt)) self.canElideLoad(body_tail) else false; const is_unused = self.liveness.isUnused(inst); - return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused); + return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused, err_cold); } - fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { const o = self.ng.object; const mod = o.pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -5998,7 +6042,7 @@ pub const FuncGen = struct { const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]); const err_union_ty = self.typeOf(extra.data.ptr).childType(mod); const is_unused = self.liveness.isUnused(inst); - return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused); + return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, err_cold); } fn lowerTry( @@ -6009,6 +6053,7 @@ pub const FuncGen = struct { operand_is_ptr: bool, can_elide_load: bool, is_unused: bool, + err_cold: bool, ) !Builder.Value { const o = fg.ng.object; const pt = o.pt; @@ -6047,9 +6092,10 @@ pub const FuncGen = struct { const return_block = try fg.wip.block(1, "TryRet"); const continue_block = try fg.wip.block(1, "TryCont"); - _ = try fg.wip.brCond(is_err, return_block, continue_block); + _ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely); fg.wip.cursor = .{ .block = return_block }; + if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold(); try fg.genBodyDebugScope(null, body); fg.wip.cursor = .{ .block = continue_block }; @@ -6076,9 +6122,11 @@ pub const FuncGen = struct { fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const o = self.ng.object; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const cond = try self.resolveInst(pl_op.operand); - const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload); + + const switch_br = self.air.unwrapSwitch(inst); + + const cond = try self.resolveInst(switch_br.operand); + const else_block = try self.wip.block(1, "Default"); const llvm_usize = try o.lowerType(Type.usize); const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) @@ -6086,34 +6134,70 @@ pub const FuncGen = struct { else cond; - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - var llvm_cases_len: u32 = 0; - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = - @ptrCast(self.air.extra[case.end..][0..case.data.items_len]); - const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len]; - extra_index = case.end + case.data.items_len + case_body.len; + const llvm_cases_len = llvm_cases_len: { + var len: u32 = 0; + var it = switch_br.iterateCases(); + while (it.next()) |case| len += @intCast(case.items.len); + break :llvm_cases_len len; + }; + + const weights: Builder.Function.Instruction.BrCond.Weights = weights: { + // First pass. If any weights are `.unpredictable`, unpredictable. + // If all are `.none` or `.cold`, none. + var any_likely = false; + for (0..switch_br.cases_len) |case_idx| { + switch (switch_br.getHint(@intCast(case_idx))) { + .none, .cold => {}, + .likely, .unlikely => any_likely = true, + .unpredictable => break :weights .unpredictable, + } + } + switch (switch_br.getElseHint()) { + .none, .cold => {}, + .likely, .unlikely => any_likely = true, + .unpredictable => break :weights .unpredictable, + } + if (!any_likely) break :weights .none; - llvm_cases_len += @intCast(items.len); - } + var weights = try self.gpa.alloc(Builder.Metadata, llvm_cases_len + 1); + defer self.gpa.free(weights); - var wip_switch = try self.wip.@"switch"(cond_int, else_block, llvm_cases_len); - defer wip_switch.finish(&self.wip); + const else_weight: u32 = switch (switch_br.getElseHint()) { + .unpredictable => unreachable, + .none, .cold => 1000, + .likely => 2000, + .unlikely => 1, + }; + weights[0] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight)); + + var weight_idx: usize = 1; + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const weight_val: u32 = switch (switch_br.getHint(case.idx)) { + .unpredictable => unreachable, + .none, .cold => 1000, + .likely => 2000, + .unlikely => 1, + }; + const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val)); + @memset(weights[weight_idx..][0..case.items.len], weight_meta); + weight_idx += case.items.len; + } + + assert(weight_idx == weights.len); - extra_index = switch_br.end; - case_i = 0; - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = - @ptrCast(self.air.extra[case.end..][0..case.data.items_len]); - const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + case.data.items_len + case_body.len; + const branch_weights_str = try o.builder.metadataString("branch_weights"); + const tuple = try o.builder.strTuple(branch_weights_str, weights); + break :weights @enumFromInt(@intFromEnum(tuple)); + }; - const case_block = try self.wip.block(@intCast(items.len), "Case"); + var wip_switch = try self.wip.@"switch"(cond_int, else_block, llvm_cases_len, weights); + defer wip_switch.finish(&self.wip); - for (items) |item| { + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const case_block = try self.wip.block(@intCast(case.items.len), "Case"); + for (case.items) |item| { const llvm_item = (try self.resolveInst(item)).toConst().?; const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder)) try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize) @@ -6121,13 +6205,14 @@ pub const FuncGen = struct { llvm_item; try wip_switch.addCase(llvm_int_item, case_block, &self.wip); } - self.wip.cursor = .{ .block = case_block }; - try self.genBodyDebugScope(null, case_body); + if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold(); + try self.genBodyDebugScope(null, case.body); } + const else_body = it.elseBody(); self.wip.cursor = .{ .block = else_block }; - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]); + if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold(); if (else_body.len != 0) { try self.genBodyDebugScope(null, else_body); } else { @@ -7758,7 +7843,7 @@ pub const FuncGen = struct { const fail_block = try fg.wip.block(1, "OverflowFail"); const ok_block = try fg.wip.block(1, "OverflowOk"); - _ = try fg.wip.brCond(overflow_bit, fail_block, ok_block); + _ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none); fg.wip.cursor = .{ .block = fail_block }; try fg.buildSimplePanic(.integer_overflow); @@ -9399,7 +9484,7 @@ pub const FuncGen = struct { self.wip.cursor = .{ .block = loop_block }; const it_ptr = try self.wip.phi(.ptr, ""); const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, ""); - _ = try self.wip.brCond(end, body_block, end_block); + _ = try self.wip.brCond(end, body_block, end_block, .none); self.wip.cursor = .{ .block = body_block }; const elem_abi_align = elem_ty.abiAlignment(pt); @@ -9437,7 +9522,7 @@ pub const FuncGen = struct { const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero); const memset_block = try self.wip.block(1, "MemsetTrapSkip"); const end_block = try self.wip.block(2, "MemsetTrapEnd"); - _ = try self.wip.brCond(cond, memset_block, end_block); + _ = try self.wip.brCond(cond, memset_block, end_block, .none); self.wip.cursor = .{ .block = memset_block }; _ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind); _ = try self.wip.br(end_block); @@ -9472,7 +9557,7 @@ pub const FuncGen = struct { const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero); const memcpy_block = try self.wip.block(1, "MemcpyTrapSkip"); const end_block = try self.wip.block(2, "MemcpyTrapEnd"); - _ = try self.wip.brCond(cond, memcpy_block, end_block); + _ = try self.wip.brCond(cond, memcpy_block, end_block, .none); self.wip.cursor = .{ .block = memcpy_block }; _ = try self.wip.callMemCpy( dest_ptr, @@ -9641,7 +9726,7 @@ pub const FuncGen = struct { const valid_block = try self.wip.block(@intCast(names.len), "Valid"); const invalid_block = try self.wip.block(1, "Invalid"); const end_block = try self.wip.block(2, "End"); - var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len)); + var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none); defer wip_switch.finish(&self.wip); for (0..names.len) |name_index| { @@ -9717,7 +9802,7 @@ pub const FuncGen = struct { const named_block = try wip.block(@intCast(enum_type.names.len), "Named"); const unnamed_block = try wip.block(1, "Unnamed"); const tag_int_value = wip.arg(0); - var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len)); + var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len), .none); defer wip_switch.finish(&wip); for (0..enum_type.names.len) |field_index| { @@ -9867,7 +9952,7 @@ pub const FuncGen = struct { const cond = try self.wip.icmp(.ult, i, llvm_vector_len, ""); const loop_then = try self.wip.block(1, "ReduceLoopThen"); - _ = try self.wip.brCond(cond, loop_then, loop_exit); + _ = try self.wip.brCond(cond, loop_then, loop_exit, .none); { self.wip.cursor = .{ .block = loop_then }; diff --git a/src/codegen/llvm/Builder.zig b/src/codegen/llvm/Builder.zig index 90da3bdd7a92..9ada51acadf5 100644 --- a/src/codegen/llvm/Builder.zig +++ b/src/codegen/llvm/Builder.zig @@ -4817,12 +4817,22 @@ pub const Function = struct { cond: Value, then: Block.Index, @"else": Block.Index, + weights: Weights, + pub const Weights = enum(u32) { + // We can do this as metadata indices 0 and 1 are reserved. + none = 0, + unpredictable = 1, + /// These values should be converted to `Metadata` to be used + /// in a `prof` annotation providing branch weights. + _, + }; }; pub const Switch = struct { val: Value, default: Block.Index, cases_len: u32, + weights: BrCond.Weights, //case_vals: [cases_len]Constant, //case_blocks: [cases_len]Block.Index, }; @@ -4969,7 +4979,8 @@ pub const Function = struct { }; pub const Info = packed struct(u32) { call_conv: CallConv, - _: u22 = undefined, + has_op_bundle_cold: bool, + _: u21 = undefined, }; }; @@ -5036,6 +5047,7 @@ pub const Function = struct { FunctionAttributes, Type, Value, + Instruction.BrCond.Weights, => @enumFromInt(value), MemoryAccessInfo, Instruction.Alloca.Info, @@ -5201,6 +5213,7 @@ pub const WipFunction = struct { cond: Value, then: Block.Index, @"else": Block.Index, + weights: enum { none, unpredictable, then_likely, else_likely }, ) Allocator.Error!Instruction.Index { assert(cond.typeOfWip(self) == .i1); try self.ensureUnusedExtraCapacity(1, Instruction.BrCond, 0); @@ -5210,6 +5223,22 @@ pub const WipFunction = struct { .cond = cond, .then = then, .@"else" = @"else", + .weights = switch (weights) { + .none => .none, + .unpredictable => .unpredictable, + .then_likely, .else_likely => w: { + const branch_weights_str = try self.builder.metadataString("branch_weights"); + const unlikely_const = try self.builder.metadataConstant(try self.builder.intConst(.i32, 1)); + const likely_const = try self.builder.metadataConstant(try self.builder.intConst(.i32, 2000)); + const weight_vals: [2]Metadata = switch (weights) { + .none, .unpredictable => unreachable, + .then_likely => .{ likely_const, unlikely_const }, + .else_likely => .{ unlikely_const, likely_const }, + }; + const tuple = try self.builder.strTuple(branch_weights_str, &weight_vals); + break :w @enumFromInt(@intFromEnum(tuple)); + }, + }, }), }); then.ptr(self).branches += 1; @@ -5248,6 +5277,7 @@ pub const WipFunction = struct { val: Value, default: Block.Index, cases_len: u32, + weights: Instruction.BrCond.Weights, ) Allocator.Error!WipSwitch { try self.ensureUnusedExtraCapacity(1, Instruction.Switch, cases_len * 2); const instruction = try self.addInst(null, .{ @@ -5256,6 +5286,7 @@ pub const WipFunction = struct { .val = val, .default = default, .cases_len = cases_len, + .weights = weights, }), }); _ = self.extra.addManyAsSliceAssumeCapacity(cases_len * 2); @@ -5895,6 +5926,20 @@ pub const WipFunction = struct { callee: Value, args: []const Value, name: []const u8, + ) Allocator.Error!Value { + return self.callInner(kind, call_conv, function_attributes, ty, callee, args, name, false); + } + + fn callInner( + self: *WipFunction, + kind: Instruction.Call.Kind, + call_conv: CallConv, + function_attributes: FunctionAttributes, + ty: Type, + callee: Value, + args: []const Value, + name: []const u8, + has_op_bundle_cold: bool, ) Allocator.Error!Value { const ret_ty = ty.functionReturn(self.builder); assert(ty.isFunction(self.builder)); @@ -5918,7 +5963,10 @@ pub const WipFunction = struct { .tail_fast => .@"tail call fast", }, .data = self.addExtraAssumeCapacity(Instruction.Call{ - .info = .{ .call_conv = call_conv }, + .info = .{ + .call_conv = call_conv, + .has_op_bundle_cold = has_op_bundle_cold, + }, .attributes = function_attributes, .ty = ty, .callee = callee, @@ -5964,6 +6012,20 @@ pub const WipFunction = struct { ); } + pub fn callIntrinsicAssumeCold(self: *WipFunction) Allocator.Error!Value { + const intrinsic = try self.builder.getIntrinsic(.assume, &.{}); + return self.callInner( + .normal, + CallConv.default, + .none, + intrinsic.typeOf(self.builder), + intrinsic.toValue(self.builder), + &.{try self.builder.intValue(.i1, 1)}, + "", + true, + ); + } + pub fn callMemCpy( self: *WipFunction, dst: Value, @@ -6040,7 +6102,7 @@ pub const WipFunction = struct { break :blk metadata; }, - .constant => |constant| try self.builder.debugConstant(constant), + .constant => |constant| try self.builder.metadataConstant(constant), .metadata => |metadata| metadata, }; } @@ -6099,6 +6161,7 @@ pub const WipFunction = struct { FunctionAttributes, Type, Value, + Instruction.BrCond.Weights, => @intFromEnum(value), MemoryAccessInfo, Instruction.Alloca.Info, @@ -6380,6 +6443,7 @@ pub const WipFunction = struct { .cond = instructions.map(extra.cond), .then = extra.then, .@"else" = extra.@"else", + .weights = extra.weights, }); }, .call, @@ -6522,6 +6586,7 @@ pub const WipFunction = struct { .val = instructions.map(extra.data.val), .default = extra.data.default, .cases_len = extra.data.cases_len, + .weights = extra.data.weights, }); wip_extra.appendSlice(case_vals); wip_extra.appendSlice(case_blocks); @@ -6744,6 +6809,7 @@ pub const WipFunction = struct { FunctionAttributes, Type, Value, + Instruction.BrCond.Weights, => @intFromEnum(value), MemoryAccessInfo, Instruction.Alloca.Info, @@ -6792,6 +6858,7 @@ pub const WipFunction = struct { FunctionAttributes, Type, Value, + Instruction.BrCond.Weights, => @enumFromInt(value), MemoryAccessInfo, Instruction.Alloca.Info, @@ -7697,6 +7764,7 @@ pub const MetadataString = enum(u32) { pub const Metadata = enum(u32) { none = 0, + empty_tuple = 1, _, const first_forward_reference = 1 << 29; @@ -7734,6 +7802,7 @@ pub const Metadata = enum(u32) { enumerator_signed_negative, subrange, tuple, + str_tuple, module_flag, expression, local_var, @@ -7779,6 +7848,7 @@ pub const Metadata = enum(u32) { .enumerator_signed_negative, .subrange, .tuple, + .str_tuple, .module_flag, .local_var, .parameter, @@ -8043,6 +8113,13 @@ pub const Metadata = enum(u32) { // elements: [elements_len]Metadata }; + pub const StrTuple = struct { + str: MetadataString, + elements_len: u32, + + // elements: [elements_len]Metadata + }; + pub const ModuleFlag = struct { behavior: Metadata, name: MetadataString, @@ -8355,7 +8432,7 @@ pub const Metadata = enum(u32) { }; pub fn init(options: Options) Allocator.Error!Builder { - var self = Builder{ + var self: Builder = .{ .gpa = options.allocator, .strip = options.strip, @@ -8454,7 +8531,9 @@ pub fn init(options: Options) Allocator.Error!Builder { assert(try self.intConst(.i32, 0) == .@"0"); assert(try self.intConst(.i32, 1) == .@"1"); assert(try self.noneConst(.token) == .none); - if (!self.strip) assert(try self.debugNone() == .none); + + assert(try self.metadataNone() == .none); + assert(try self.metadataTuple(&.{}) == .empty_tuple); try self.metadata_string_indices.append(self.gpa, 0); assert(try self.metadataString("") == .none); @@ -9683,6 +9762,13 @@ pub fn printUnbuffered( extra.then.toInst(&function).fmt(function_index, self), extra.@"else".toInst(&function).fmt(function_index, self), }); + switch (extra.weights) { + .none => {}, + .unpredictable => try writer.writeAll(", !unpredictable !{}"), + _ => try writer.print("{}", .{ + try metadata_formatter.fmt(", !prof ", @as(Metadata, @enumFromInt(@intFromEnum(extra.weights)))), + }), + } }, .call, .@"call fast", @@ -9727,6 +9813,9 @@ pub fn printUnbuffered( }); } try writer.writeByte(')'); + if (extra.data.info.has_op_bundle_cold) { + try writer.writeAll(" [ \"cold\"() ]"); + } const call_function_attributes = extra.data.attributes.func(self); if (call_function_attributes != .none) try writer.print(" #{d}", .{ (try attribute_groups.getOrPutValue( @@ -9937,6 +10026,13 @@ pub fn printUnbuffered( }, ); try writer.writeAll(" ]"); + switch (extra.data.weights) { + .none => {}, + .unpredictable => try writer.writeAll(", !unpredictable !{}"), + _ => try writer.print("{}", .{ + try metadata_formatter.fmt(", !prof ", @as(Metadata, @enumFromInt(@intFromEnum(extra.data.weights)))), + }), + } }, .va_arg => |tag| { const extra = function.extraData(Function.Instruction.VaArg, instruction.data); @@ -10285,6 +10381,17 @@ pub fn printUnbuffered( }); try writer.writeAll("}\n"); }, + .str_tuple => { + var extra = self.metadataExtraDataTrail(Metadata.StrTuple, metadata_item.data); + const elements = extra.trail.next(extra.data.elements_len, Metadata, self); + try writer.print("!{{{[str]%}", .{ + .str = try metadata_formatter.fmt("", extra.data.str), + }); + for (elements) |element| try writer.print("{[element]%}", .{ + .element = try metadata_formatter.fmt("", element), + }); + try writer.writeAll("}\n"); + }, .module_flag => { const extra = self.metadataExtraData(Metadata.ModuleFlag, metadata_item.data); try writer.print("!{{{[behavior]%}{[name]%}{[constant]%}}}\n", .{ @@ -11797,9 +11904,9 @@ pub fn debugNamed(self: *Builder, name: MetadataString, operands: []const Metada self.debugNamedAssumeCapacity(name, operands); } -fn debugNone(self: *Builder) Allocator.Error!Metadata { +fn metadataNone(self: *Builder) Allocator.Error!Metadata { try self.ensureUnusedMetadataCapacity(1, NoExtra, 0); - return self.debugNoneAssumeCapacity(); + return self.metadataNoneAssumeCapacity(); } pub fn debugFile( @@ -12088,12 +12195,21 @@ pub fn debugExpression( return self.debugExpressionAssumeCapacity(elements); } -pub fn debugTuple( +pub fn metadataTuple( self: *Builder, elements: []const Metadata, ) Allocator.Error!Metadata { try self.ensureUnusedMetadataCapacity(1, Metadata.Tuple, elements.len); - return self.debugTupleAssumeCapacity(elements); + return self.metadataTupleAssumeCapacity(elements); +} + +pub fn strTuple( + self: *Builder, + str: MetadataString, + elements: []const Metadata, +) Allocator.Error!Metadata { + try self.ensureUnusedMetadataCapacity(1, Metadata.StrTuple, elements.len); + return self.strTupleAssumeCapacity(str, elements); } pub fn debugModuleFlag( @@ -12164,9 +12280,9 @@ pub fn debugGlobalVarExpression( return self.debugGlobalVarExpressionAssumeCapacity(variable, expression); } -pub fn debugConstant(self: *Builder, value: Constant) Allocator.Error!Metadata { +pub fn metadataConstant(self: *Builder, value: Constant) Allocator.Error!Metadata { try self.ensureUnusedMetadataCapacity(1, NoExtra, 0); - return self.debugConstantAssumeCapacity(value); + return self.metadataConstantAssumeCapacity(value); } pub fn debugForwardReferenceSetType(self: *Builder, fwd_ref: Metadata, ty: Metadata) void { @@ -12261,8 +12377,7 @@ fn debugNamedAssumeCapacity(self: *Builder, name: MetadataString, operands: []co }; } -pub fn debugNoneAssumeCapacity(self: *Builder) Metadata { - assert(!self.strip); +pub fn metadataNoneAssumeCapacity(self: *Builder) Metadata { return self.metadataSimpleAssumeCapacity(.none, .{}); } @@ -12738,11 +12853,10 @@ fn debugExpressionAssumeCapacity( return @enumFromInt(gop.index); } -fn debugTupleAssumeCapacity( +fn metadataTupleAssumeCapacity( self: *Builder, elements: []const Metadata, ) Metadata { - assert(!self.strip); const Key = struct { elements: []const Metadata, }; @@ -12785,6 +12899,55 @@ fn debugTupleAssumeCapacity( return @enumFromInt(gop.index); } +fn strTupleAssumeCapacity( + self: *Builder, + str: MetadataString, + elements: []const Metadata, +) Metadata { + const Key = struct { + str: MetadataString, + elements: []const Metadata, + }; + const Adapter = struct { + builder: *const Builder, + pub fn hash(_: @This(), key: Key) u32 { + var hasher = comptime std.hash.Wyhash.init(std.hash.uint32(@intFromEnum(Metadata.Tag.tuple))); + hasher.update(std.mem.sliceAsBytes(key.elements)); + return @truncate(hasher.final()); + } + + pub fn eql(ctx: @This(), lhs_key: Key, _: void, rhs_index: usize) bool { + if (.str_tuple != ctx.builder.metadata_items.items(.tag)[rhs_index]) return false; + const rhs_data = ctx.builder.metadata_items.items(.data)[rhs_index]; + var rhs_extra = ctx.builder.metadataExtraDataTrail(Metadata.StrTuple, rhs_data); + return rhs_extra.data.str == lhs_key.str and std.mem.eql( + Metadata, + lhs_key.elements, + rhs_extra.trail.next(rhs_extra.data.elements_len, Metadata, ctx.builder), + ); + } + }; + + const gop = self.metadata_map.getOrPutAssumeCapacityAdapted( + Key{ .str = str, .elements = elements }, + Adapter{ .builder = self }, + ); + + if (!gop.found_existing) { + gop.key_ptr.* = {}; + gop.value_ptr.* = {}; + self.metadata_items.appendAssumeCapacity(.{ + .tag = .str_tuple, + .data = self.addMetadataExtraAssumeCapacity(Metadata.StrTuple{ + .str = str, + .elements_len = @intCast(elements.len), + }), + }); + self.metadata_extra.appendSliceAssumeCapacity(@ptrCast(elements)); + } + return @enumFromInt(gop.index); +} + fn debugModuleFlagAssumeCapacity( self: *Builder, behavior: Metadata, @@ -12875,8 +13038,7 @@ fn debugGlobalVarExpressionAssumeCapacity( }); } -fn debugConstantAssumeCapacity(self: *Builder, constant: Constant) Metadata { - assert(!self.strip); +fn metadataConstantAssumeCapacity(self: *Builder, constant: Constant) Metadata { const Adapter = struct { builder: *const Builder, pub fn hash(_: @This(), key: Constant) u32 { @@ -13755,15 +13917,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co } // METADATA_KIND_BLOCK - if (!self.strip) { + { const MetadataKindBlock = ir.MetadataKindBlock; var metadata_kind_block = try module_block.enterSubBlock(MetadataKindBlock, true); - inline for (@typeInfo(ir.MetadataKind).Enum.fields) |field| { - try metadata_kind_block.writeAbbrev(MetadataKindBlock.Kind{ - .id = field.value, - .name = field.name, - }); + inline for (@typeInfo(ir.FixedMetadataKind).Enum.fields) |field| { + // don't include `dbg` in stripped functions + if (!(self.strip and std.mem.eql(u8, field.name, "dbg"))) { + try metadata_kind_block.writeAbbrev(MetadataKindBlock.Kind{ + .id = field.value, + .name = field.name, + }); + } } try metadata_kind_block.end(); @@ -13808,14 +13973,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co const metadata_adapter = MetadataAdapter.init(self, constant_adapter); // METADATA_BLOCK - if (!self.strip) { + { const MetadataBlock = ir.MetadataBlock; var metadata_block = try module_block.enterSubBlock(MetadataBlock, true); const MetadataBlockWriter = @TypeOf(metadata_block); // Emit all MetadataStrings - { + if (self.metadata_string_map.count() > 1) { const strings_offset, const strings_size = blk: { var strings_offset: u32 = 0; var strings_size: u32 = 0; @@ -14046,7 +14211,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co else -%val << 1 | 1); } - try metadata_block.writeUnabbrev(MetadataBlock.Enumerator.id, record.items); + try metadata_block.writeUnabbrev(@intFromEnum(MetadataBlock.Enumerator.id), record.items); continue; }; try metadata_block.writeAbbrevAdapted(MetadataBlock.Enumerator{ @@ -14085,6 +14250,22 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .elements = elements, }, metadata_adapter); }, + .str_tuple => { + var extra = self.metadataExtraDataTrail(Metadata.StrTuple, data); + + const elements = extra.trail.next(extra.data.elements_len, Metadata, self); + + const all_elems = try self.gpa.alloc(Metadata, elements.len + 1); + defer self.gpa.free(all_elems); + all_elems[0] = @enumFromInt(metadata_adapter.getMetadataStringIndex(extra.data.str)); + for (elements, all_elems[1..]) |elem, *out_elem| { + out_elem.* = @enumFromInt(metadata_adapter.getMetadataIndex(elem)); + } + + try metadata_block.writeAbbrev(MetadataBlock.Node{ + .elements = all_elems, + }); + }, .module_flag => { const extra = self.metadataExtraData(Metadata.ModuleFlag, data); try metadata_block.writeAbbrev(MetadataBlock.Node{ @@ -14177,7 +14358,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co try metadata_block.writeAbbrev(MetadataBlock.GlobalDeclAttachment{ .value = @enumFromInt(constant_adapter.getConstantIndex(global.toConst())), - .kind = ir.MetadataKind.dbg, + .kind = .dbg, .metadata = @enumFromInt(metadata_adapter.getMetadataIndex(global_ptr.dbg) - 1), }); } @@ -14186,6 +14367,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co try metadata_block.end(); } + // OPERAND_BUNDLE_TAGS_BLOCK + { + const OperandBundleTags = ir.OperandBundleTags; + var operand_bundle_tags_block = try module_block.enterSubBlock(OperandBundleTags, true); + + try operand_bundle_tags_block.writeAbbrev(OperandBundleTags.OperandBundleTag{ + .tag = "cold", + }); + + try operand_bundle_tags_block.end(); + } + // Block info { const BlockInfo = ir.BlockInfo; @@ -14220,20 +14413,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co constant_adapter: ConstantAdapter, metadata_adapter: MetadataAdapter, func: *const Function, - instruction_index: u32 = 0, - - pub fn init( - const_adapter: ConstantAdapter, - meta_adapter: MetadataAdapter, - func: *const Function, - ) @This() { - return .{ - .constant_adapter = const_adapter, - .metadata_adapter = meta_adapter, - .func = func, - .instruction_index = 0, - }; - } + instruction_index: Function.Instruction.Index, pub fn get(adapter: @This(), value: anytype, comptime field_name: []const u8) @TypeOf(value) { _ = field_name; @@ -14254,7 +14434,6 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .instruction => |instruction| instruction.valueIndex(adapter.func) + adapter.firstInstr(), .constant => |constant| adapter.constant_adapter.getConstantIndex(constant), .metadata => |metadata| { - assert(!adapter.func.strip); const real_metadata = metadata.unwrap(adapter.metadata_adapter.builder); if (@intFromEnum(real_metadata) < Metadata.first_local_metadata) return adapter.metadata_adapter.getMetadataIndex(real_metadata) - 1; @@ -14282,19 +14461,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co } pub fn offset(adapter: @This()) u32 { - return @as( - Function.Instruction.Index, - @enumFromInt(adapter.instruction_index), - ).valueIndex(adapter.func) + adapter.firstInstr(); + return adapter.instruction_index.valueIndex(adapter.func) + adapter.firstInstr(); } fn firstInstr(adapter: @This()) u32 { return adapter.constant_adapter.numConstants(); } - - pub fn next(adapter: *@This()) void { - adapter.instruction_index += 1; - } }; for (self.functions.items, 0..) |func, func_index| { @@ -14307,7 +14479,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co try function_block.writeAbbrev(FunctionBlock.DeclareBlocks{ .num_blocks = func.blocks.len }); - var adapter = FunctionAdapter.init(constant_adapter, metadata_adapter, &func); + var adapter: FunctionAdapter = .{ + .constant_adapter = constant_adapter, + .metadata_adapter = metadata_adapter, + .func = &func, + .instruction_index = @enumFromInt(0), + }; // Emit function level metadata block if (!func.strip and func.debug_values.len > 0) { @@ -14330,21 +14507,27 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co var has_location = false; var block_incoming_len: u32 = undefined; - for (0..func.instructions.len) |instr_index| { - const tag = tags[instr_index]; - + for (tags, datas, 0..) |tag, data, instr_index| { + adapter.instruction_index = @enumFromInt(instr_index); record.clearRetainingCapacity(); switch (tag) { - .block => block_incoming_len = datas[instr_index], - .arg => {}, + .arg => continue, + .block => { + block_incoming_len = data; + continue; + }, .@"unreachable" => try function_block.writeAbbrev(FunctionBlock.Unreachable{}), .call, .@"musttail call", .@"notail call", .@"tail call", => |kind| { - var extra = func.extraDataTrail(Function.Instruction.Call, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.Call, data); + + if (extra.data.info.has_op_bundle_cold) { + try function_block.writeAbbrev(FunctionBlock.ColdOperandBundle{}); + } const call_conv = extra.data.info.call_conv; const args = extra.trail.next(extra.data.args_len, Value, &func); @@ -14367,7 +14550,11 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .@"notail call fast", .@"tail call fast", => |kind| { - var extra = func.extraDataTrail(Function.Instruction.Call, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.Call, data); + + if (extra.data.info.has_op_bundle_cold) { + try function_block.writeAbbrev(FunctionBlock.ColdOperandBundle{}); + } const call_conv = extra.data.info.call_conv; const args = extra.trail.next(extra.data.args_len, Value, &func); @@ -14405,7 +14592,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .srem, .ashr, => |kind| { - const extra = func.extraData(Function.Instruction.Binary, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Binary, data); try function_block.writeAbbrev(FunctionBlock.Binary{ .opcode = kind.toBinaryOpcode(), .lhs = adapter.getOffsetValueIndex(extra.lhs), @@ -14417,7 +14604,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .@"lshr exact", .@"ashr exact", => |kind| { - const extra = func.extraData(Function.Instruction.Binary, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Binary, data); try function_block.writeAbbrev(FunctionBlock.BinaryExact{ .opcode = kind.toBinaryOpcode(), .lhs = adapter.getOffsetValueIndex(extra.lhs), @@ -14437,7 +14624,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .@"shl nuw", .@"shl nuw nsw", => |kind| { - const extra = func.extraData(Function.Instruction.Binary, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Binary, data); try function_block.writeAbbrev(FunctionBlock.BinaryNoWrap{ .opcode = kind.toBinaryOpcode(), .lhs = adapter.getOffsetValueIndex(extra.lhs), @@ -14468,7 +14655,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .@"frem fast", .@"fsub fast", => |kind| { - const extra = func.extraData(Function.Instruction.Binary, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Binary, data); try function_block.writeAbbrev(FunctionBlock.BinaryFast{ .opcode = kind.toBinaryOpcode(), .lhs = adapter.getOffsetValueIndex(extra.lhs), @@ -14479,7 +14666,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .alloca, .@"alloca inalloca", => |kind| { - const extra = func.extraData(Function.Instruction.Alloca, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Alloca, data); const alignment = extra.info.alignment.toLlvm(); try function_block.writeAbbrev(FunctionBlock.Alloca{ .inst_type = extra.type, @@ -14508,7 +14695,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .sext, .zext, => |kind| { - const extra = func.extraData(Function.Instruction.Cast, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Cast, data); try function_block.writeAbbrev(FunctionBlock.Cast{ .val = adapter.getOffsetValueIndex(extra.val), .type_index = extra.type, @@ -14542,7 +14729,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .@"icmp ule", .@"icmp ult", => |kind| { - const extra = func.extraData(Function.Instruction.Binary, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Binary, data); try function_block.writeAbbrev(FunctionBlock.Cmp{ .lhs = adapter.getOffsetValueIndex(extra.lhs), .rhs = adapter.getOffsetValueIndex(extra.rhs), @@ -14566,7 +14753,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .@"fcmp fast une", .@"fcmp fast uno", => |kind| { - const extra = func.extraData(Function.Instruction.Binary, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Binary, data); try function_block.writeAbbrev(FunctionBlock.CmpFast{ .lhs = adapter.getOffsetValueIndex(extra.lhs), .rhs = adapter.getOffsetValueIndex(extra.rhs), @@ -14575,14 +14762,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .fneg => try function_block.writeAbbrev(FunctionBlock.FNeg{ - .val = adapter.getOffsetValueIndex(@enumFromInt(datas[instr_index])), + .val = adapter.getOffsetValueIndex(@enumFromInt(data)), }), .@"fneg fast" => try function_block.writeAbbrev(FunctionBlock.FNegFast{ - .val = adapter.getOffsetValueIndex(@enumFromInt(datas[instr_index])), + .val = adapter.getOffsetValueIndex(@enumFromInt(data)), .fast_math = FastMath.fast, }), .extractvalue => { - var extra = func.extraDataTrail(Function.Instruction.ExtractValue, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.ExtractValue, data); const indices = extra.trail.next(extra.data.indices_len, u32, &func); try function_block.writeAbbrev(FunctionBlock.ExtractValue{ .val = adapter.getOffsetValueIndex(extra.data.val), @@ -14590,7 +14777,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .insertvalue => { - var extra = func.extraDataTrail(Function.Instruction.InsertValue, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.InsertValue, data); const indices = extra.trail.next(extra.data.indices_len, u32, &func); try function_block.writeAbbrev(FunctionBlock.InsertValue{ .val = adapter.getOffsetValueIndex(extra.data.val), @@ -14599,14 +14786,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .extractelement => { - const extra = func.extraData(Function.Instruction.ExtractElement, datas[instr_index]); + const extra = func.extraData(Function.Instruction.ExtractElement, data); try function_block.writeAbbrev(FunctionBlock.ExtractElement{ .val = adapter.getOffsetValueIndex(extra.val), .index = adapter.getOffsetValueIndex(extra.index), }); }, .insertelement => { - const extra = func.extraData(Function.Instruction.InsertElement, datas[instr_index]); + const extra = func.extraData(Function.Instruction.InsertElement, data); try function_block.writeAbbrev(FunctionBlock.InsertElement{ .val = adapter.getOffsetValueIndex(extra.val), .elem = adapter.getOffsetValueIndex(extra.elem), @@ -14614,7 +14801,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .select => { - const extra = func.extraData(Function.Instruction.Select, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Select, data); try function_block.writeAbbrev(FunctionBlock.Select{ .lhs = adapter.getOffsetValueIndex(extra.lhs), .rhs = adapter.getOffsetValueIndex(extra.rhs), @@ -14622,7 +14809,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .@"select fast" => { - const extra = func.extraData(Function.Instruction.Select, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Select, data); try function_block.writeAbbrev(FunctionBlock.SelectFast{ .lhs = adapter.getOffsetValueIndex(extra.lhs), .rhs = adapter.getOffsetValueIndex(extra.rhs), @@ -14631,7 +14818,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .shufflevector => { - const extra = func.extraData(Function.Instruction.ShuffleVector, datas[instr_index]); + const extra = func.extraData(Function.Instruction.ShuffleVector, data); try function_block.writeAbbrev(FunctionBlock.ShuffleVector{ .lhs = adapter.getOffsetValueIndex(extra.lhs), .rhs = adapter.getOffsetValueIndex(extra.rhs), @@ -14641,7 +14828,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .getelementptr, .@"getelementptr inbounds", => |kind| { - var extra = func.extraDataTrail(Function.Instruction.GetElementPtr, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.GetElementPtr, data); const indices = extra.trail.next(extra.data.indices_len, Value, &func); try function_block.writeAbbrevAdapted( FunctionBlock.GetElementPtr{ @@ -14654,7 +14841,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co ); }, .load => { - const extra = func.extraData(Function.Instruction.Load, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Load, data); try function_block.writeAbbrev(FunctionBlock.Load{ .ptr = adapter.getOffsetValueIndex(extra.ptr), .ty = extra.type, @@ -14663,7 +14850,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .@"load atomic" => { - const extra = func.extraData(Function.Instruction.Load, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Load, data); try function_block.writeAbbrev(FunctionBlock.LoadAtomic{ .ptr = adapter.getOffsetValueIndex(extra.ptr), .ty = extra.type, @@ -14674,7 +14861,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .store => { - const extra = func.extraData(Function.Instruction.Store, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Store, data); try function_block.writeAbbrev(FunctionBlock.Store{ .ptr = adapter.getOffsetValueIndex(extra.ptr), .val = adapter.getOffsetValueIndex(extra.val), @@ -14683,7 +14870,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .@"store atomic" => { - const extra = func.extraData(Function.Instruction.Store, datas[instr_index]); + const extra = func.extraData(Function.Instruction.Store, data); try function_block.writeAbbrev(FunctionBlock.StoreAtomic{ .ptr = adapter.getOffsetValueIndex(extra.ptr), .val = adapter.getOffsetValueIndex(extra.val), @@ -14695,11 +14882,11 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }, .br => { try function_block.writeAbbrev(FunctionBlock.BrUnconditional{ - .block = datas[instr_index], + .block = data, }); }, .br_cond => { - const extra = func.extraData(Function.Instruction.BrCond, datas[instr_index]); + const extra = func.extraData(Function.Instruction.BrCond, data); try function_block.writeAbbrev(FunctionBlock.BrConditional{ .then_block = @intFromEnum(extra.then), .else_block = @intFromEnum(extra.@"else"), @@ -14707,7 +14894,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .@"switch" => { - var extra = func.extraDataTrail(Function.Instruction.Switch, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.Switch, data); try record.ensureUnusedCapacity(self.gpa, 3 + extra.data.cases_len * 2); @@ -14730,7 +14917,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co try function_block.writeUnabbrev(12, record.items); }, .va_arg => { - const extra = func.extraData(Function.Instruction.VaArg, datas[instr_index]); + const extra = func.extraData(Function.Instruction.VaArg, data); try function_block.writeAbbrev(FunctionBlock.VaArg{ .list_type = extra.list.typeOf(@enumFromInt(func_index), self), .list = adapter.getOffsetValueIndex(extra.list), @@ -14740,7 +14927,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .phi, .@"phi fast", => |kind| { - var extra = func.extraDataTrail(Function.Instruction.Phi, datas[instr_index]); + var extra = func.extraDataTrail(Function.Instruction.Phi, data); const vals = extra.trail.next(block_incoming_len, Value, &func); const blocks = extra.trail.next(block_incoming_len, Function.Block.Index, &func); @@ -14764,11 +14951,11 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co try function_block.writeUnabbrev(16, record.items); }, .ret => try function_block.writeAbbrev(FunctionBlock.Ret{ - .val = adapter.getOffsetValueIndex(@enumFromInt(datas[instr_index])), + .val = adapter.getOffsetValueIndex(@enumFromInt(data)), }), .@"ret void" => try function_block.writeAbbrev(FunctionBlock.RetVoid{}), .atomicrmw => { - const extra = func.extraData(Function.Instruction.AtomicRmw, datas[instr_index]); + const extra = func.extraData(Function.Instruction.AtomicRmw, data); try function_block.writeAbbrev(FunctionBlock.AtomicRmw{ .ptr = adapter.getOffsetValueIndex(extra.ptr), .val = adapter.getOffsetValueIndex(extra.val), @@ -14782,7 +14969,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co .cmpxchg, .@"cmpxchg weak", => |kind| { - const extra = func.extraData(Function.Instruction.CmpXchg, datas[instr_index]); + const extra = func.extraData(Function.Instruction.CmpXchg, data); try function_block.writeAbbrev(FunctionBlock.CmpXchg{ .ptr = adapter.getOffsetValueIndex(extra.ptr), @@ -14797,7 +14984,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co }); }, .fence => { - const info: MemoryAccessInfo = @bitCast(datas[instr_index]); + const info: MemoryAccessInfo = @bitCast(data); try function_block.writeAbbrev(FunctionBlock.Fence{ .ordering = info.success_ordering, .sync_scope = info.sync_scope, @@ -14806,7 +14993,7 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co } if (!func.strip) { - if (func.debug_locations.get(@enumFromInt(instr_index))) |debug_location| { + if (func.debug_locations.get(adapter.instruction_index)) |debug_location| { switch (debug_location) { .no_location => has_location = false, .location => |location| { @@ -14823,8 +15010,6 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co try function_block.writeAbbrev(FunctionBlock.DebugLocAgain{}); } } - - adapter.next(); } // VALUE_SYMTAB @@ -14850,18 +15035,48 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co } // METADATA_ATTACHMENT_BLOCK - if (!func.strip) blk: { - const dbg = func.global.ptrConst(self).dbg; - - if (dbg == .none) break :blk; - + { const MetadataAttachmentBlock = ir.MetadataAttachmentBlock; var metadata_attach_block = try function_block.enterSubBlock(MetadataAttachmentBlock, false); - try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentSingle{ - .kind = ir.MetadataKind.dbg, - .metadata = @enumFromInt(metadata_adapter.getMetadataIndex(dbg) - 1), - }); + dbg: { + if (func.strip) break :dbg; + const dbg = func.global.ptrConst(self).dbg; + if (dbg == .none) break :dbg; + try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentGlobalSingle{ + .kind = .dbg, + .metadata = @enumFromInt(metadata_adapter.getMetadataIndex(dbg) - 1), + }); + } + + var instr_index: u32 = 0; + for (func.instructions.items(.tag), func.instructions.items(.data)) |instr_tag, data| switch (instr_tag) { + .arg, .block => {}, // not an actual instruction + else => { + instr_index += 1; + }, + .br_cond, .@"switch" => { + const weights = switch (instr_tag) { + .br_cond => func.extraData(Function.Instruction.BrCond, data).weights, + .@"switch" => func.extraData(Function.Instruction.Switch, data).weights, + else => unreachable, + }; + switch (weights) { + .none => {}, + .unpredictable => try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{ + .inst = instr_index, + .kind = .unpredictable, + .metadata = @enumFromInt(metadata_adapter.getMetadataIndex(.empty_tuple) - 1), + }), + _ => try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{ + .inst = instr_index, + .kind = .prof, + .metadata = @enumFromInt(metadata_adapter.getMetadataIndex(@enumFromInt(@intFromEnum(weights))) - 1), + }), + } + instr_index += 1; + }, + }; try metadata_attach_block.end(); } diff --git a/src/codegen/llvm/ir.zig b/src/codegen/llvm/ir.zig index 6a7c6c6857db..4d7effdaaf94 100644 --- a/src/codegen/llvm/ir.zig +++ b/src/codegen/llvm/ir.zig @@ -20,8 +20,142 @@ const ColumnAbbrev = AbbrevOp{ .vbr = 8 }; const BlockAbbrev = AbbrevOp{ .vbr = 6 }; -pub const MetadataKind = enum(u1) { +/// Unused tags are commented out so that they are omitted in the generated +/// bitcode, which scans over this enum using reflection. +pub const FixedMetadataKind = enum(u8) { dbg = 0, + //tbaa = 1, + prof = 2, + //fpmath = 3, + //range = 4, + //@"tbaa.struct" = 5, + //@"invariant.load" = 6, + //@"alias.scope" = 7, + //@"noalias" = 8, + //nontemporal = 9, + //@"llvm.mem.parallel_loop_access" = 10, + //nonnull = 11, + //dereferenceable = 12, + //dereferenceable_or_null = 13, + //@"make.implicit" = 14, + unpredictable = 15, + //@"invariant.group" = 16, + //@"align" = 17, + //@"llvm.loop" = 18, + //type = 19, + //section_prefix = 20, + //absolute_symbol = 21, + //associated = 22, + //callees = 23, + //irr_loop = 24, + //@"llvm.access.group" = 25, + //callback = 26, + //@"llvm.preserve.access.index" = 27, + //vcall_visibility = 28, + //noundef = 29, + //annotation = 30, + //nosanitize = 31, + //func_sanitize = 32, + //exclude = 33, + //memprof = 34, + //callsite = 35, + //kcfi_type = 36, + //pcsections = 37, + //DIAssignID = 38, + //@"coro.outside.frame" = 39, +}; + +pub const MetadataCode = enum(u8) { + /// MDSTRING: [values] + STRING_OLD = 1, + /// VALUE: [type num, value num] + VALUE = 2, + /// NODE: [n x md num] + NODE = 3, + /// STRING: [values] + NAME = 4, + /// DISTINCT_NODE: [n x md num] + DISTINCT_NODE = 5, + /// [n x [id, name]] + KIND = 6, + /// [distinct, line, col, scope, inlined-at?] + LOCATION = 7, + /// OLD_NODE: [n x (type num, value num)] + OLD_NODE = 8, + /// OLD_FN_NODE: [n x (type num, value num)] + OLD_FN_NODE = 9, + /// NAMED_NODE: [n x mdnodes] + NAMED_NODE = 10, + /// [m x [value, [n x [id, mdnode]]] + ATTACHMENT = 11, + /// [distinct, tag, vers, header, n x md num] + GENERIC_DEBUG = 12, + /// [distinct, count, lo] + SUBRANGE = 13, + /// [isUnsigned|distinct, value, name] + ENUMERATOR = 14, + /// [distinct, tag, name, size, align, enc] + BASIC_TYPE = 15, + /// [distinct, filename, directory, checksumkind, checksum] + FILE = 16, + /// [distinct, ...] + DERIVED_TYPE = 17, + /// [distinct, ...] + COMPOSITE_TYPE = 18, + /// [distinct, flags, types, cc] + SUBROUTINE_TYPE = 19, + /// [distinct, ...] + COMPILE_UNIT = 20, + /// [distinct, ...] + SUBPROGRAM = 21, + /// [distinct, scope, file, line, column] + LEXICAL_BLOCK = 22, + ///[distinct, scope, file, discriminator] + LEXICAL_BLOCK_FILE = 23, + /// [distinct, scope, file, name, line, exportSymbols] + NAMESPACE = 24, + /// [distinct, scope, name, type, ...] + TEMPLATE_TYPE = 25, + /// [distinct, scope, name, type, value, ...] + TEMPLATE_VALUE = 26, + /// [distinct, ...] + GLOBAL_VAR = 27, + /// [distinct, ...] + LOCAL_VAR = 28, + /// [distinct, n x element] + EXPRESSION = 29, + /// [distinct, name, file, line, ...] + OBJC_PROPERTY = 30, + /// [distinct, tag, scope, entity, line, name] + IMPORTED_ENTITY = 31, + /// [distinct, scope, name, ...] + MODULE = 32, + /// [distinct, macinfo, line, name, value] + MACRO = 33, + /// [distinct, macinfo, line, file, ...] + MACRO_FILE = 34, + /// [count, offset] blob([lengths][chars]) + STRINGS = 35, + /// [valueid, n x [id, mdnode]] + GLOBAL_DECL_ATTACHMENT = 36, + /// [distinct, var, expr] + GLOBAL_VAR_EXPR = 37, + /// [offset] + INDEX_OFFSET = 38, + /// [bitpos] + INDEX = 39, + /// [distinct, scope, name, file, line] + LABEL = 40, + /// [distinct, name, size, align,...] + STRING_TYPE = 41, + /// [distinct, scope, name, variable,...] + COMMON_BLOCK = 44, + /// [distinct, count, lo, up, stride] + GENERIC_SUBRANGE = 45, + /// [n x [type num, value num]] + ARG_LIST = 46, + /// [distinct, ...] + ASSIGN_ID = 47, }; pub const Identification = struct { @@ -622,16 +756,29 @@ pub const MetadataAttachmentBlock = struct { pub const id = 16; pub const abbrevs = [_]type{ - AttachmentSingle, + AttachmentGlobalSingle, + AttachmentInstructionSingle, }; - pub const AttachmentSingle = struct { + pub const AttachmentGlobalSingle = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 11 }, + .{ .literal = @intFromEnum(MetadataCode.ATTACHMENT) }, .{ .fixed = 1 }, MetadataAbbrev, }; - kind: MetadataKind, + kind: FixedMetadataKind, + metadata: Builder.Metadata, + }; + + pub const AttachmentInstructionSingle = struct { + pub const ops = [_]AbbrevOp{ + .{ .literal = @intFromEnum(MetadataCode.ATTACHMENT) }, + ValueAbbrev, + .{ .fixed = 5 }, + MetadataAbbrev, + }; + inst: u32, + kind: FixedMetadataKind, metadata: Builder.Metadata, }; }; @@ -666,7 +813,7 @@ pub const MetadataBlock = struct { pub const Strings = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 35 }, + .{ .literal = @intFromEnum(MetadataCode.STRINGS) }, .{ .vbr = 6 }, .{ .vbr = 6 }, .blob, @@ -678,7 +825,7 @@ pub const MetadataBlock = struct { pub const File = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 16 }, + .{ .literal = @intFromEnum(MetadataCode.FILE) }, .{ .literal = 0 }, // is distinct MetadataAbbrev, // filename MetadataAbbrev, // directory @@ -692,7 +839,7 @@ pub const MetadataBlock = struct { pub const CompileUnit = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 20 }, + .{ .literal = @intFromEnum(MetadataCode.COMPILE_UNIT) }, .{ .literal = 1 }, // is distinct .{ .literal = std.dwarf.LANG.C99 }, // source language MetadataAbbrev, // file @@ -726,7 +873,7 @@ pub const MetadataBlock = struct { pub const Subprogram = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 21 }, + .{ .literal = @intFromEnum(MetadataCode.SUBPROGRAM) }, .{ .literal = 0b111 }, // is distinct | has sp flags | has flags MetadataAbbrev, // scope MetadataAbbrev, // name @@ -763,7 +910,7 @@ pub const MetadataBlock = struct { pub const LexicalBlock = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 22 }, + .{ .literal = @intFromEnum(MetadataCode.LEXICAL_BLOCK) }, .{ .literal = 0 }, // is distinct MetadataAbbrev, // scope MetadataAbbrev, // file @@ -779,7 +926,7 @@ pub const MetadataBlock = struct { pub const Location = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 7 }, + .{ .literal = @intFromEnum(MetadataCode.LOCATION) }, .{ .literal = 0 }, // is distinct LineAbbrev, // line ColumnAbbrev, // column @@ -796,7 +943,7 @@ pub const MetadataBlock = struct { pub const BasicType = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 15 }, + .{ .literal = @intFromEnum(MetadataCode.BASIC_TYPE) }, .{ .literal = 0 }, // is distinct .{ .literal = std.dwarf.TAG.base_type }, // tag MetadataAbbrev, // name @@ -813,7 +960,7 @@ pub const MetadataBlock = struct { pub const CompositeType = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 18 }, + .{ .literal = @intFromEnum(MetadataCode.COMPOSITE_TYPE) }, .{ .literal = 0 | 0x2 }, // is distinct | is not used in old type ref .{ .fixed = 32 }, // tag MetadataAbbrev, // name @@ -852,7 +999,7 @@ pub const MetadataBlock = struct { pub const DerivedType = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 17 }, + .{ .literal = @intFromEnum(MetadataCode.DERIVED_TYPE) }, .{ .literal = 0 }, // is distinct .{ .fixed = 32 }, // tag MetadataAbbrev, // name @@ -880,7 +1027,7 @@ pub const MetadataBlock = struct { pub const SubroutineType = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 19 }, + .{ .literal = @intFromEnum(MetadataCode.SUBROUTINE_TYPE) }, .{ .literal = 0 | 0x2 }, // is distinct | has no old type refs .{ .literal = 0 }, // flags MetadataAbbrev, // types @@ -891,7 +1038,7 @@ pub const MetadataBlock = struct { }; pub const Enumerator = struct { - pub const id = 14; + pub const id: MetadataCode = .ENUMERATOR; pub const Flags = packed struct(u3) { distinct: bool = false, @@ -900,7 +1047,7 @@ pub const MetadataBlock = struct { }; pub const ops = [_]AbbrevOp{ - .{ .literal = Enumerator.id }, + .{ .literal = @intFromEnum(Enumerator.id) }, .{ .fixed = @bitSizeOf(Flags) }, // flags .{ .vbr = 6 }, // bit width MetadataAbbrev, // name @@ -915,7 +1062,7 @@ pub const MetadataBlock = struct { pub const Subrange = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 13 }, + .{ .literal = @intFromEnum(MetadataCode.SUBRANGE) }, .{ .literal = 0b10 }, // is distinct | version MetadataAbbrev, // count MetadataAbbrev, // lower bound @@ -929,7 +1076,7 @@ pub const MetadataBlock = struct { pub const Expression = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 29 }, + .{ .literal = @intFromEnum(MetadataCode.EXPRESSION) }, .{ .literal = 0 | (3 << 1) }, // is distinct | version MetadataArrayAbbrev, // elements }; @@ -939,7 +1086,7 @@ pub const MetadataBlock = struct { pub const Node = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 3 }, + .{ .literal = @intFromEnum(MetadataCode.NODE) }, MetadataArrayAbbrev, // elements }; @@ -948,7 +1095,7 @@ pub const MetadataBlock = struct { pub const LocalVar = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 28 }, + .{ .literal = @intFromEnum(MetadataCode.LOCAL_VAR) }, .{ .literal = 0b10 }, // is distinct | has alignment MetadataAbbrev, // scope MetadataAbbrev, // name @@ -970,7 +1117,7 @@ pub const MetadataBlock = struct { pub const Parameter = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 28 }, + .{ .literal = @intFromEnum(MetadataCode.LOCAL_VAR) }, .{ .literal = 0b10 }, // is distinct | has alignment MetadataAbbrev, // scope MetadataAbbrev, // name @@ -993,7 +1140,7 @@ pub const MetadataBlock = struct { pub const GlobalVar = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 27 }, + .{ .literal = @intFromEnum(MetadataCode.GLOBAL_VAR) }, .{ .literal = 0b101 }, // is distinct | version MetadataAbbrev, // scope MetadataAbbrev, // name @@ -1020,7 +1167,7 @@ pub const MetadataBlock = struct { pub const GlobalVarExpression = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 37 }, + .{ .literal = @intFromEnum(MetadataCode.GLOBAL_VAR_EXPR) }, .{ .literal = 0 }, // is distinct MetadataAbbrev, // variable MetadataAbbrev, // expression @@ -1032,7 +1179,7 @@ pub const MetadataBlock = struct { pub const Constant = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 2 }, + .{ .literal = @intFromEnum(MetadataCode.VALUE) }, MetadataAbbrev, // type MetadataAbbrev, // value }; @@ -1043,7 +1190,7 @@ pub const MetadataBlock = struct { pub const Name = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 4 }, + .{ .literal = @intFromEnum(MetadataCode.NAME) }, .{ .array_fixed = 8 }, // name }; @@ -1052,7 +1199,7 @@ pub const MetadataBlock = struct { pub const NamedNode = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 10 }, + .{ .literal = @intFromEnum(MetadataCode.NAMED_NODE) }, MetadataArrayAbbrev, // elements }; @@ -1061,18 +1208,32 @@ pub const MetadataBlock = struct { pub const GlobalDeclAttachment = struct { pub const ops = [_]AbbrevOp{ - .{ .literal = 36 }, + .{ .literal = @intFromEnum(MetadataCode.GLOBAL_DECL_ATTACHMENT) }, ValueAbbrev, // value id .{ .fixed = 1 }, // kind MetadataAbbrev, // elements }; value: Builder.Constant, - kind: MetadataKind, + kind: FixedMetadataKind, metadata: Builder.Metadata, }; }; +pub const OperandBundleTags = struct { + pub const id = 21; + + pub const abbrevs = [_]type{OperandBundleTag}; + + pub const OperandBundleTag = struct { + pub const ops = [_]AbbrevOp{ + .{ .literal = 1 }, + .array_char6, + }; + tag: []const u8, + }; +}; + pub const FunctionMetadataBlock = struct { pub const id = 15; @@ -1132,6 +1293,7 @@ pub const FunctionBlock = struct { Fence, DebugLoc, DebugLocAgain, + ColdOperandBundle, }; pub const DeclareBlocks = struct { @@ -1644,6 +1806,13 @@ pub const FunctionBlock = struct { .{ .literal = 33 }, }; }; + + pub const ColdOperandBundle = struct { + pub const ops = [_]AbbrevOp{ + .{ .literal = 55 }, + .{ .literal = 0 }, + }; + }; }; pub const FunctionValueSymbolTable = struct { diff --git a/src/codegen/spirv.zig b/src/codegen/spirv.zig index a89dd8f10bff..16b77d74f591 100644 --- a/src/codegen/spirv.zig +++ b/src/codegen/spirv.zig @@ -410,7 +410,7 @@ const NavGen = struct { } pub fn fail(self: *NavGen, comptime format: []const u8, args: anytype) Error { - @setCold(true); + @branchHint(.cold); const zcu = self.pt.zcu; const src_loc = zcu.navSrcLoc(self.owner_nav); assert(self.error_msg == null); @@ -6172,11 +6172,10 @@ const NavGen = struct { const pt = self.pt; const mod = pt.zcu; const target = self.getTarget(); - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const cond_ty = self.typeOf(pl_op.operand); - const cond = try self.resolve(pl_op.operand); + const switch_br = self.air.unwrapSwitch(inst); + const cond_ty = self.typeOf(switch_br.operand); + const cond = try self.resolve(switch_br.operand); var cond_indirect = try self.convertToIndirect(cond_ty, cond); - const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload); const cond_words: u32 = switch (cond_ty.zigTypeTag(mod)) { .Bool, .ErrorSet => 1, @@ -6203,18 +6202,15 @@ const NavGen = struct { else => return self.todo("implement switch for type {s}", .{@tagName(cond_ty.zigTypeTag(mod))}), }; - const num_cases = switch_br.data.cases_len; + const num_cases = switch_br.cases_len; // Compute the total number of arms that we need. // Zig switches are grouped by condition, so we need to loop through all of them const num_conditions = blk: { - var extra_index: usize = switch_br.end; var num_conditions: u32 = 0; - for (0..num_cases) |_| { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const case_body = self.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]; - extra_index = case.end + case.data.items_len + case_body.len; - num_conditions += case.data.items_len; + var it = switch_br.iterateCases(); + while (it.next()) |case| { + num_conditions += @intCast(case.items.len); } break :blk num_conditions; }; @@ -6243,17 +6239,12 @@ const NavGen = struct { // Emit each of the cases { - var extra_index: usize = switch_br.end; - for (0..num_cases) |case_i| { + var it = switch_br.iterateCases(); + while (it.next()) |case| { // SPIR-V needs a literal here, which' width depends on the case condition. - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]); - const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len]; - extra_index = case.end + case.data.items_len + case_body.len; - - const label = case_labels.at(case_i); + const label = case_labels.at(case.idx); - for (items) |item| { + for (case.items) |item| { const value = (try self.air.value(item, pt)) orelse unreachable; const int_val: u64 = switch (cond_ty.zigTypeTag(mod)) { .Bool, .Int => if (cond_ty.isSignedInt(mod)) @bitCast(value.toSignedInt(pt)) else value.toUnsignedInt(pt), @@ -6284,20 +6275,15 @@ const NavGen = struct { } // Now, finally, we can start emitting each of the cases. - var extra_index: usize = switch_br.end; - for (0..num_cases) |case_i| { - const case = self.air.extraData(Air.SwitchBr.Case, extra_index); - const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]); - const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + case.data.items_len + case_body.len; - - const label = case_labels.at(case_i); + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const label = case_labels.at(case.idx); try self.beginSpvBlock(label); switch (self.control_flow) { .structured => { - const next_block = try self.genStructuredBody(.selection, case_body); + const next_block = try self.genStructuredBody(.selection, case.body); incoming_structured_blocks.appendAssumeCapacity(.{ .src_label = self.current_block_label, .next_block = next_block, @@ -6305,12 +6291,12 @@ const NavGen = struct { try self.func.body.emitBranch(self.spv.gpa, merge_label.?); }, .unstructured => { - try self.genBody(case_body); + try self.genBody(case.body); }, } } - const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]); + const else_body = it.elseBody(); try self.beginSpvBlock(default); if (else_body.len != 0) { switch (self.control_flow) { diff --git a/src/crash_report.zig b/src/crash_report.zig index 67ec0e0eb046..c0a90d870572 100644 --- a/src/crash_report.zig +++ b/src/crash_report.zig @@ -153,8 +153,8 @@ fn writeFilePath(file: *Zcu.File, writer: anytype) !void { } pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn { + @branchHint(.cold); PanicSwitch.preDispatch(); - @setCold(true); const ret_addr = maybe_ret_addr orelse @returnAddress(); const stack_ctx: StackContext = .{ .current = .{ .ret_addr = ret_addr } }; PanicSwitch.dispatch(error_return_trace, stack_ctx, msg); diff --git a/src/print_air.zig b/src/print_air.zig index 4db66a1d3c45..a3565cdb4052 100644 --- a/src/print_air.zig +++ b/src/print_air.zig @@ -297,8 +297,8 @@ const Writer = struct { .union_init => try w.writeUnionInit(s, inst), .br => try w.writeBr(s, inst), .cond_br => try w.writeCondBr(s, inst), - .@"try" => try w.writeTry(s, inst), - .try_ptr => try w.writeTryPtr(s, inst), + .@"try", .try_cold => try w.writeTry(s, inst), + .try_ptr, .try_ptr_cold => try w.writeTryPtr(s, inst), .switch_br => try w.writeSwitchBr(s, inst), .cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst), .fence => try w.writeFence(s, inst), @@ -825,41 +825,40 @@ const Writer = struct { } fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { - const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const switch_br = w.air.extraData(Air.SwitchBr, pl_op.payload); + const switch_br = w.air.unwrapSwitch(inst); + const liveness = if (w.liveness) |liveness| - liveness.getSwitchBr(w.gpa, inst, switch_br.data.cases_len + 1) catch + liveness.getSwitchBr(w.gpa, inst, switch_br.cases_len + 1) catch @panic("out of memory") else blk: { - const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.data.cases_len + 1) catch + const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.cases_len + 1) catch @panic("out of memory"); @memset(slice, &.{}); break :blk Liveness.SwitchBrTable{ .deaths = slice }; }; defer w.gpa.free(liveness.deaths); - var extra_index: usize = switch_br.end; - var case_i: u32 = 0; - try w.writeOperand(s, inst, 0, pl_op.operand); + try w.writeOperand(s, inst, 0, switch_br.operand); if (w.skip_body) return s.writeAll(", ..."); const old_indent = w.indent; w.indent += 2; - while (case_i < switch_br.data.cases_len) : (case_i += 1) { - const case = w.air.extraData(Air.SwitchBr.Case, extra_index); - const items = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[case.end..][0..case.data.items_len])); - const case_body: []const Air.Inst.Index = @ptrCast(w.air.extra[case.end + items.len ..][0..case.data.body_len]); - extra_index = case.end + case.data.items_len + case_body.len; - + var it = switch_br.iterateCases(); + while (it.next()) |case| { try s.writeAll(", ["); - for (items, 0..) |item, item_i| { + for (case.items, 0..) |item, item_i| { if (item_i != 0) try s.writeAll(", "); try w.writeInstRef(s, item, false); } - try s.writeAll("] => {\n"); + try s.writeAll("] "); + const hint = switch_br.getHint(case.idx); + if (hint != .none) { + try s.print(".{s} ", .{@tagName(hint)}); + } + try s.writeAll("=> {\n"); w.indent += 2; - const deaths = liveness.deaths[case_i]; + const deaths = liveness.deaths[case.idx]; if (deaths.len != 0) { try s.writeByteNTimes(' ', w.indent); for (deaths, 0..) |operand, i| { @@ -869,15 +868,20 @@ const Writer = struct { try s.writeAll("\n"); } - try w.writeBody(s, case_body); + try w.writeBody(s, case.body); w.indent -= 2; try s.writeByteNTimes(' ', w.indent); try s.writeAll("}"); } - const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra_index..][0..switch_br.data.else_body_len]); + const else_body = it.elseBody(); if (else_body.len != 0) { - try s.writeAll(", else => {\n"); + try s.writeAll(", else "); + const hint = switch_br.getElseHint(); + if (hint != .none) { + try s.print(".{s} ", .{@tagName(hint)}); + } + try s.writeAll("=> {\n"); w.indent += 2; const deaths = liveness.deaths[liveness.deaths.len - 1]; diff --git a/src/print_zir.zig b/src/print_zir.zig index 774b222626dd..885549ba0c78 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -565,7 +565,6 @@ const Writer = struct { .fence, .set_float_mode, .set_align_stack, - .set_cold, .wasm_memory_size, .int_from_error, .error_from_int, @@ -574,6 +573,7 @@ const Writer = struct { .work_item_id, .work_group_size, .work_group_id, + .branch_hint, => { const inst_data = self.code.extraData(Zir.Inst.UnNode, extended.operand).data; try self.writeInstRef(stream, inst_data.operand); diff --git a/stage1/zig1.wasm b/stage1/zig1.wasm index 0b0b5d4de020..eb4f74c44190 100644 Binary files a/stage1/zig1.wasm and b/stage1/zig1.wasm differ diff --git a/test/behavior.zig b/test/behavior.zig index ea8ea713aca9..430c34bcc6d5 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -12,6 +12,7 @@ test { _ = @import("behavior/bitcast.zig"); _ = @import("behavior/bitreverse.zig"); _ = @import("behavior/bool.zig"); + _ = @import("behavior/builtin_functions_returning_void_or_noreturn.zig"); _ = @import("behavior/byteswap.zig"); _ = @import("behavior/byval_arg_var.zig"); _ = @import("behavior/call.zig"); diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig index 9d3474260468..d08494ba6496 100644 --- a/test/behavior/basic.zig +++ b/test/behavior/basic.zig @@ -107,13 +107,90 @@ test "non const ptr to aliased type" { try expect(?*int == ?*i32); } -test "cold function" { - thisIsAColdFn(); - comptime thisIsAColdFn(); +test "function branch hints" { + const S = struct { + fn none() void { + @branchHint(.none); + } + fn likely() void { + @branchHint(.likely); + } + fn unlikely() void { + @branchHint(.unlikely); + } + fn cold() void { + @branchHint(.cold); + } + fn unpredictable() void { + @branchHint(.unpredictable); + } + }; + S.none(); + S.likely(); + S.unlikely(); + S.cold(); + S.unpredictable(); + comptime S.none(); + comptime S.likely(); + comptime S.unlikely(); + comptime S.cold(); + comptime S.unpredictable(); +} + +test "if branch hints" { + var t: bool = undefined; + t = true; + if (t) { + @branchHint(.likely); + } else { + @branchHint(.cold); + } } -fn thisIsAColdFn() void { - @setCold(true); +test "switch branch hints" { + var t: bool = undefined; + t = true; + switch (t) { + true => { + @branchHint(.likely); + }, + false => { + @branchHint(.cold); + }, + } +} + +test "orelse branch hints" { + var x: ?u32 = undefined; + x = 123; + const val = x orelse val: { + @branchHint(.cold); + break :val 456; + }; + try expect(val == 123); +} + +test "catch branch hints" { + var x: error{Bad}!u32 = undefined; + x = 123; + const val = x catch val: { + @branchHint(.cold); + break :val 456; + }; + try expect(val == 123); +} + +test "and/or branch hints" { + var t: bool = undefined; + t = true; + try expect(t or b: { + @branchHint(.unlikely); + break :b false; + }); + try expect(t and b: { + @branchHint(.likely); + break :b true; + }); } test "unicode escape in character literal" { diff --git a/test/behavior/builtin_functions_returning_void_or_noreturn.zig b/test/behavior/builtin_functions_returning_void_or_noreturn.zig index 92617ef3a427..bb4908dd0d51 100644 --- a/test/behavior/builtin_functions_returning_void_or_noreturn.zig +++ b/test/behavior/builtin_functions_returning_void_or_noreturn.zig @@ -22,7 +22,6 @@ test { try testing.expectEqual(noreturn, @TypeOf(if (true) @panic("") else {})); try testing.expectEqual({}, @prefetch(&val, .{})); try testing.expectEqual({}, @setAlignStack(16)); - try testing.expectEqual({}, @setCold(true)); try testing.expectEqual({}, @setEvalBranchQuota(0)); try testing.expectEqual({}, @setFloatMode(.optimized)); try testing.expectEqual({}, @setRuntimeSafety(true)); diff --git a/test/cases/compile_errors/function-only_builtins_outside_function.zig b/test/cases/compile_errors/function-only_builtins_outside_function.zig index 3484b5ca4665..a9871e2b5956 100644 --- a/test/cases/compile_errors/function-only_builtins_outside_function.zig +++ b/test/cases/compile_errors/function-only_builtins_outside_function.zig @@ -3,7 +3,7 @@ comptime { } comptime { - @setCold(true); + @branchHint(.cold); } comptime { @@ -55,7 +55,7 @@ comptime { // target=native // // :2:5: error: '@setAlignStack' outside function scope -// :6:5: error: '@setCold' outside function scope +// :6:5: error: '@branchHint' outside function scope // :10:5: error: '@src' outside function scope // :14:5: error: '@returnAddress' outside function scope // :18:5: error: '@frameAddress' outside function scope diff --git a/test/cases/compile_errors/invalid_branch_hint.zig b/test/cases/compile_errors/invalid_branch_hint.zig new file mode 100644 index 000000000000..1ceb8a686e5b --- /dev/null +++ b/test/cases/compile_errors/invalid_branch_hint.zig @@ -0,0 +1,41 @@ +const globl = g: { + @branchHint(.none); + break :g {}; +}; + +comptime { + @branchHint(.none); +} + +test { + @branchHint(.none); +} + +export fn foo() void { + { + @branchHint(.none); + } +} + +export fn bar() void { + _ = (b: { + @branchHint(.none); + break :b true; + }) or true; +} + +export fn qux() void { + (b: { + @branchHint(.none); + break :b @as(?void, {}); + }) orelse unreachable; +} + +// error +// +// :2:5: error: '@branchHint' outside function scope +// :7:5: error: '@branchHint' outside function scope +// :11:5: error: '@branchHint' must appear as the first statement in a function or conditional branch +// :16:9: error: '@branchHint' must appear as the first statement in a function or conditional branch +// :22:9: error: '@branchHint' must appear as the first statement in a function or conditional branch +// :29:9: error: '@branchHint' must appear as the first statement in a function or conditional branch