perf(codegen): Restore noundef On PassMode::Cast Args In Rust ABI

compiler/rustc_target/src/callconv/mod.rs

@@ -1,8 +1,8 @@
 use std::{fmt, iter};
 
 use rustc_abi::{
-    AddressSpace, Align, BackendRepr, CanonAbi, ExternAbi, HasDataLayout, Primitive, Reg, RegKind,
-    Scalar, Size, TyAbiInterface, TyAndLayout,
+    AddressSpace, Align, BackendRepr, CanonAbi, ExternAbi, FieldsShape, HasDataLayout, Primitive,
+    Reg, RegKind, Scalar, Size, TyAbiInterface, TyAndLayout, Variants,
 };
 use rustc_macros::HashStable_Generic;
 
@@ -514,6 +514,11 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32: false };
     }
 
+    pub fn cast_to_with_attrs<T: Into<CastTarget>>(&mut self, target: T, attrs: ArgAttributes) {
+        self.mode =
+            PassMode::Cast { cast: Box::new(target.into().with_attrs(attrs)), pad_i32: false };
+    }
+
     pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32: bool) {
         self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32 };
     }
@@ -801,7 +806,12 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                         // We want to pass small aggregates as immediates, but using
                         // an LLVM aggregate type for this leads to bad optimizations,
                         // so we pick an appropriately sized integer type instead.
-                        arg.cast_to(Reg { kind: RegKind::Integer, size });
+                        let attr = if layout_is_noundef(arg.layout, cx) {
+                            ArgAttribute::NoUndef
+                        } else {
+                            ArgAttribute::default()
+                        };
+                        arg.cast_to_with_attrs(Reg { kind: RegKind::Integer, size }, attr.into());
                     }
                 }
 
@@ -836,6 +846,66 @@ impl<'a, Ty> FnAbi<'a, Ty> {
     }
 }
 
+/// Determines whether `layout` contains no uninit bytes (no padding, no unions),
+/// using only the computed layout.
+///
+/// Conservative: returns `false` for anything it cannot prove fully initialized,
+/// including multi-variant enums and SIMD vectors.
+// FIXME: extend to multi-variant enums (per-variant padding analysis needed).
+fn layout_is_noundef<'a, Ty, C>(layout: TyAndLayout<'a, Ty>, cx: &C) -> bool
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+    C: HasDataLayout,
+{
+    match layout.backend_repr {
+        BackendRepr::Scalar(scalar) => !scalar.is_uninit_valid(),
+        BackendRepr::ScalarPair(s1, s2) => {
+            !s1.is_uninit_valid()
+                && !s2.is_uninit_valid()
+                // Ensure there is no padding.
+                && s1.size(cx) + s2.size(cx) == layout.size
+        }
+        BackendRepr::Memory { .. } => match layout.fields {
+            FieldsShape::Primitive | FieldsShape::Union(_) => false,
+            // Array elements are at stride offsets with no inter-element gaps.
+            FieldsShape::Array { stride: _, count } => {
+                count == 0 || layout_is_noundef(layout.field(cx, 0), cx)
+            }
+            FieldsShape::Arbitrary { .. } => {
+                // With `Variants::Multiple`, `layout.fields` only covers shared
+                // bytes (niche/discriminant); per-variant data is absent, so
+                // full coverage cannot be proven.
+                matches!(layout.variants, Variants::Single { .. }) && fields_are_noundef(layout, cx)
+            }
+        },
+        BackendRepr::SimdVector { .. } | BackendRepr::ScalableVector { .. } => false,
+    }
+}
+
+/// Returns `true` if the fields of `layout` contiguously cover bytes `0..layout.size`
+/// with no padding gaps and each field is recursively `layout_is_noundef`.
+fn fields_are_noundef<'a, Ty, C>(layout: TyAndLayout<'a, Ty>, cx: &C) -> bool
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+    C: HasDataLayout,
+{
+    let mut cursor = Size::ZERO;
+    for i in layout.fields.index_by_increasing_offset() {
+        let field = layout.field(cx, i);
+        if field.size == Size::ZERO {
+            continue;
+        }
+        if layout.fields.offset(i) != cursor {
+            return false;
+        }
+        if !layout_is_noundef(field, cx) {
+            return false;
+        }
+        cursor += field.size;
+    }
+    cursor == layout.size
+}
+
 // Some types are used a lot. Make sure they don't unintentionally get bigger.
 #[cfg(target_pointer_width = "64")]
 mod size_asserts {

compiler/rustc_ty_utils/src/abi.rs

@@ -627,6 +627,7 @@ fn fn_abi_adjust_for_abi<'tcx>(
 
     if abi.is_rustic_abi() {
         fn_abi.adjust_for_rust_abi(cx);
+
         // Look up the deduced parameter attributes for this function, if we have its def ID and
         // we're optimizing in non-incremental mode. We'll tag its parameters with those attributes
         // as appropriate.

tests/codegen-llvm/abi-noundef-cast.rs

@@ -0,0 +1,206 @@
+// Verify that `PassMode::Cast` arguments/returns in the Rust ABI carry `noundef`
+// when the original layout provably contains no uninit bytes, and correctly omit
+// it when uninit bytes or padding may be present.
+//
+// See <https://github.com/rust-lang/rust/issues/123183>.
+
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
+//@ only-64bit
+
+#![crate_type = "lib"]
+
+use std::mem::MaybeUninit;
+
+// CHECK-LABEL: @arg_array_u32x2(
+// CHECK-SAME: i64 noundef
+#[no_mangle]
+pub fn arg_array_u32x2(v: [u32; 2]) -> u32 {
+    v[0]
+}
+
+// CHECK-LABEL: @arg_array_u8x4(
+// CHECK-SAME: i32 noundef
+#[no_mangle]
+pub fn arg_array_u8x4(v: [u8; 4]) -> u8 {
+    v[0]
+}
+
+// CHECK-LABEL: @arg_nested_array(
+// CHECK-SAME: i64 noundef
+#[no_mangle]
+pub fn arg_nested_array(v: [[u8; 2]; 4]) -> u8 {
+    v[0][0]
+}
+
+// CHECK-LABEL: @arg_array_bool(
+// CHECK-SAME: i64 noundef
+#[no_mangle]
+pub fn arg_array_bool(v: [bool; 8]) -> bool {
+    v[0]
+}
+
+struct FourU8 {
+    a: u8,
+    b: u8,
+    c: u8,
+    d: u8,
+}
+
+// CHECK-LABEL: @arg_four_u8(
+// CHECK-SAME: i32 noundef
+#[no_mangle]
+pub fn arg_four_u8(v: FourU8) -> u8 {
+    v.a
+}
+
+struct Wrapper([u32; 2]);
+
+// CHECK-LABEL: @arg_newtype_wrapper(
+// CHECK-SAME: i64 noundef
+#[no_mangle]
+pub fn arg_newtype_wrapper(v: Wrapper) -> u32 {
+    (v.0)[0]
+}
+
+enum SingleVariant {
+    Only([u32; 2]),
+}
+
+// CHECK-LABEL: @arg_single_variant_enum(
+// CHECK-SAME: i64 noundef
+#[no_mangle]
+pub fn arg_single_variant_enum(v: SingleVariant) -> u32 {
+    match v {
+        SingleVariant::Only(a) => a[0],
+    }
+}
+
+struct ContainsScalarPair {
+    a: (u16, u16),
+    b: u32,
+}
+
+// CHECK-LABEL: @arg_contains_scalar_pair(
+// CHECK-SAME: i64 noundef
+#[no_mangle]
+pub fn arg_contains_scalar_pair(v: ContainsScalarPair) -> u32 {
+    v.b
+}
+
+// CHECK: define noundef i64 @ret_array_u32x2(
+#[no_mangle]
+pub fn ret_array_u32x2(x: u32, y: u32) -> [u32; 2] {
+    [x, y]
+}
+
+// CHECK-LABEL: @arg_maybeuninit_u8x8(
+// CHECK-SAME: i64 %
+#[no_mangle]
+pub fn arg_maybeuninit_u8x8(v: [MaybeUninit<u8>; 8]) -> MaybeUninit<u8> {
+    v[0]
+}
+
+enum MultiVariant {
+    A(u8),
+    B(u16),
+    C,
+}
+
+// CHECK-LABEL: @arg_multi_variant_enum(
+// CHECK-SAME: i32 %
+#[no_mangle]
+pub fn arg_multi_variant_enum(v: MultiVariant) -> u8 {
+    match v {
+        MultiVariant::A(x) => x,
+        MultiVariant::B(_) | MultiVariant::C => 0,
+    }
+}
+
+#[repr(C)]
+struct HasFieldGap {
+    a: u8,
+    b: u16,
+    c: u8,
+}
+
+// CHECK-LABEL: @arg_struct_field_gap(
+// CHECK-SAME: i48 %
+#[no_mangle]
+pub fn arg_struct_field_gap(v: HasFieldGap) -> u8 {
+    v.a
+}
+
+#[repr(C)]
+struct HasPaddedPairField {
+    a: (u8, u16),
+    b: u8,
+}
+
+// CHECK-LABEL: @arg_struct_padded_pair_field(
+// CHECK-SAME: i48 %
+#[no_mangle]
+pub fn arg_struct_padded_pair_field(v: HasPaddedPairField) -> u8 {
+    v.b
+}
+
+#[repr(C)]
+struct HasUndefPairField {
+    a: (MaybeUninit<u16>, u16),
+    b: u32,
+}
+
+// CHECK-LABEL: @arg_struct_undef_pair_field(
+// CHECK-SAME: i64 %
+#[no_mangle]
+pub fn arg_struct_undef_pair_field(v: HasUndefPairField) -> u32 {
+    v.b
+}
+
+// CHECK-LABEL: @arg_triple_maybeuninit_u8(
+// CHECK-SAME: i24 %
+#[no_mangle]
+pub fn arg_triple_maybeuninit_u8(
+    v: (MaybeUninit<u8>, MaybeUninit<u8>, MaybeUninit<u8>),
+) -> MaybeUninit<u8> {
+    v.0
+}
+
+#[repr(C)]
+struct HasTrailingPadding {
+    x: u32,
+    y: u16,
+    z: u8,
+}
+
+// CHECK-LABEL: @arg_struct_trailing_pad(
+// CHECK-SAME: i64 %
+#[no_mangle]
+pub fn arg_struct_trailing_pad(v: HasTrailingPadding) -> u32 {
+    v.x
+}
+
+// CHECK-LABEL: @arg_tuple_i8_i16(
+// CHECK-SAME: i8 noundef
+// CHECK-SAME: i16 noundef
+#[no_mangle]
+pub fn arg_tuple_i8_i16(v: (i8, i16)) -> i8 {
+    v.0
+}
+
+// CHECK-LABEL: @arg_tuple_i16_maybeuninit(
+// CHECK-SAME: i16 noundef
+// CHECK-SAME: i16 %
+#[no_mangle]
+pub fn arg_tuple_i16_maybeuninit(v: (i16, MaybeUninit<i16>)) -> i16 {
+    v.0
+}
+
+// CHECK-LABEL: @arg_result_i32(
+// CHECK-SAME: i32 noundef
+// CHECK-SAME: i32 noundef
+#[no_mangle]
+pub fn arg_result_i32(v: Result<i32, i32>) -> i32 {
+    match v {
+        Ok(x) | Err(x) => x,
+    }
+}

tests/codegen-llvm/debuginfo-dse.rs

@@ -148,7 +148,7 @@ fn direct(
 // Arguments are passed through registers, the final values are poison.
 #[unsafe(no_mangle)]
 fn cast(aggregate_4xi8: Aggregate_4xi8) {
-    // CHECK-LABEL: define{{( dso_local)?}} void @cast(i32 %0)
+    // CHECK-LABEL: define{{( dso_local)?}} void @cast(i32 noundef %0)
     // CHECK: call void @opaque_fn()
     opaque_fn();
     // The temporary allocated variable is eliminated.

tests/ui/abi/pass-indirectly-attr.stderr

@@ -143,7 +143,7 @@ error: fn_abi_of(extern_rust) = FnAbi {
                                is_consecutive: false,
                            },
                            attrs: ArgAttributes {
-                               regular: ,
+                               regular: NoUndef,
                                arg_ext: None,
                                pointee_size: Size(0 bytes),
                                pointee_align: None,