Unrolled build for #143410

Rollup merge of #143410 - scottmcm:redo-transmute-again, r=RalfJung,workingjubilee

Block SIMD in transmute_immediate; delete `OperandValueKind`

Vectors have been causing me problems for years in this code, for example #110021 (comment) and #143194

See conversation in <https://rust-lang.zulipchat.com/#narrow/channel/131828-t-compiler/topic/Is.20transmuting.20a.20.60T.60.20to.20.60Tx1.60.20.28one-element.20SIMD.20vector.29.20UB.3F/near/526262799>.

By blocking SIMD in `transmute_immediate` it can be simplified to just take the `Scalar`s involved -- the backend types can be gotten from those `Scalar`s, rather than needing to be passed.  And there's an assert added to ICE it if it does get hit.

Accordingly, this changes `rvalue_creates_operand` to not send SIMD transmutes through the operand path, but to always go through memory instead, like they did back before #108442.

And thanks to those changes, I could also remove the `OperandValueKind` type that I added back then which `@RalfJung` rightly considers pretty sketchy.

cc `@folkertdev` `@workingjubilee` from the zulip conversation too

Author: rust-timer

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -13,7 +13,7 @@ use rustc_session::config::OptLevel;
 use tracing::{debug, instrument};
 
 use super::place::{PlaceRef, PlaceValue};
-use super::rvalue::transmute_immediate;
+use super::rvalue::transmute_scalar;
 use super::{FunctionCx, LocalRef};
 use crate::common::IntPredicate;
 use crate::traits::*;
@@ -346,14 +346,16 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 
         let val = if field.is_zst() {
             OperandValue::ZeroSized
+        } else if let BackendRepr::SimdVector { .. } = self.layout.backend_repr {
+            // codegen_transmute_operand doesn't support SIMD, but since the previous
+            // check handled ZSTs, the only possible field access into something SIMD
+            // is to the `non_1zst_field` that's the same SIMD. (Other things, even
+            // just padding, would change the wrapper's representation type.)
+            assert_eq!(field.size, self.layout.size);
+            self.val
         } else if field.size == self.layout.size {
             assert_eq!(offset.bytes(), 0);
-            fx.codegen_transmute_operand(bx, *self, field).unwrap_or_else(|| {
-                bug!(
-                    "Expected `codegen_transmute_operand` to handle equal-size \
-                      field {i:?} projection from {self:?} to {field:?}"
-                )
-            })
+            fx.codegen_transmute_operand(bx, *self, field)
         } else {
             let (in_scalar, imm) = match (self.val, self.layout.backend_repr) {
                 // Extract a scalar component from a pair.
@@ -613,10 +615,8 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> {
         };
 
         let mut update = |tgt: &mut Result<V, abi::Scalar>, src, from_scalar| {
-            let from_bty = bx.cx().type_from_scalar(from_scalar);
             let to_scalar = tgt.unwrap_err();
-            let to_bty = bx.cx().type_from_scalar(to_scalar);
-            let imm = transmute_immediate(bx, src, from_scalar, from_bty, to_scalar, to_bty);
+            let imm = transmute_scalar(bx, src, from_scalar, to_scalar);
             *tgt = Ok(imm);
         };
 

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -9,17 +9,20 @@ use std::mem::transmute;
 // CHECK-LABEL: @check_sse_pair_to_avx(
 #[no_mangle]
 pub unsafe fn check_sse_pair_to_avx(x: (__m128i, __m128i)) -> __m256i {
+    // CHECK: start:
     // CHECK-NOT: alloca
-    // CHECK: %0 = load <4 x i64>, ptr %x, align 16
-    // CHECK: store <4 x i64> %0, ptr %_0, align 32
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %_0, ptr align 16 %x, i64 32, i1 false)
+    // CHECK-NEXT: ret void
     transmute(x)
 }
 
 // CHECK-LABEL: @check_sse_pair_from_avx(
 #[no_mangle]
 pub unsafe fn check_sse_pair_from_avx(x: __m256i) -> (__m128i, __m128i) {
+    // CHECK: start:
     // CHECK-NOT: alloca
-    // CHECK: %0 = load <4 x i64>, ptr %x, align 32
-    // CHECK: store <4 x i64> %0, ptr %_0, align 16
+    // CHECK-NEXT: %[[TEMP:.+]] = load <4 x i64>, ptr %x, align 32
+    // CHECK-NEXT: store <4 x i64> %[[TEMP]], ptr %_0, align 16
+    // CHECK-NEXT: ret void
     transmute(x)
 }

tests/codegen/intrinsics/transmute-x64.rs

@@ -29,28 +29,28 @@ pub struct Aggregate8(u8);
 // CHECK-LABEL: @check_bigger_size(
 #[no_mangle]
 pub unsafe fn check_bigger_size(x: u16) -> u32 {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
 // CHECK-LABEL: @check_smaller_size(
 #[no_mangle]
 pub unsafe fn check_smaller_size(x: u32) -> u16 {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
 // CHECK-LABEL: @check_smaller_array(
 #[no_mangle]
 pub unsafe fn check_smaller_array(x: [u32; 7]) -> [u32; 3] {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
 // CHECK-LABEL: @check_bigger_array(
 #[no_mangle]
 pub unsafe fn check_bigger_array(x: [u32; 3]) -> [u32; 7] {
-    // CHECK: call void @llvm.trap
+    // CHECK: call void @llvm.assume(i1 false)
     transmute_unchecked(x)
 }
 
@@ -73,9 +73,9 @@ pub unsafe fn check_to_empty_array(x: [u32; 5]) -> [u32; 0] {
 #[no_mangle]
 #[custom_mir(dialect = "runtime", phase = "optimized")]
 pub unsafe fn check_from_empty_array(x: [u32; 0]) -> [u32; 5] {
-    // CHECK-NOT: trap
-    // CHECK: call void @llvm.trap
-    // CHECK-NOT: trap
+    // CHECK-NOT: call
+    // CHECK: call void @llvm.assume(i1 false)
+    // CHECK-NOT: call
     mir! {
         {
             RET = CastTransmute(x);

tests/codegen/intrinsics/transmute.rs

@@ -40,8 +40,7 @@ pub fn build_array_s(x: [f32; 4]) -> S<4> {
 // CHECK-LABEL: @build_array_transmute_s
 #[no_mangle]
 pub fn build_array_transmute_s(x: [f32; 4]) -> S<4> {
-    // CHECK: %[[VAL:.+]] = load <4 x float>, ptr %x, align [[ARRAY_ALIGN]]
-    // CHECK: store <4 x float> %[[VAL:.+]], ptr %_0, align [[VECTOR_ALIGN]]
+    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false)
     unsafe { std::mem::transmute(x) }
 }
 
@@ -55,7 +54,6 @@ pub fn build_array_t(x: [f32; 4]) -> T {
 // CHECK-LABEL: @build_array_transmute_t
 #[no_mangle]
 pub fn build_array_transmute_t(x: [f32; 4]) -> T {
-    // CHECK: %[[VAL:.+]] = load <4 x float>, ptr %x, align [[ARRAY_ALIGN]]
-    // CHECK: store <4 x float> %[[VAL:.+]], ptr %_0, align [[VECTOR_ALIGN]]
+    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}} align [[VECTOR_ALIGN]] {{.*}} align [[ARRAY_ALIGN]] {{.*}}, [[USIZE]] 16, i1 false)
     unsafe { std::mem::transmute(x) }
 }

tests/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs

@@ -111,8 +111,11 @@ pub fn fake_bool_unsigned_to_bool(b: FakeBoolUnsigned) -> bool {
 struct S([i64; 1]);
 
 // CHECK-LABEL: define{{.*}}i64 @single_element_simd_to_scalar(<1 x i64> %b)
-// CHECK: bitcast <1 x i64> %b to i64
-// CHECK: ret i64
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[RET:.+]] = alloca [8 x i8]
+// CHECK-NEXT: store <1 x i64> %b, ptr %[[RET]]
+// CHECK-NEXT: %[[TEMP:.+]] = load i64, ptr %[[RET]]
+// CHECK-NEXT: ret i64 %[[TEMP]]
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
@@ -124,8 +127,11 @@ pub extern "C" fn single_element_simd_to_scalar(b: S) -> i64 {
 }
 
 // CHECK-LABEL: define{{.*}}<1 x i64> @scalar_to_single_element_simd(i64 %b)
-// CHECK: bitcast i64 %b to <1 x i64>
-// CHECK: ret <1 x i64>
+// CHECK-NEXT: start:
+// CHECK-NEXT: %[[RET:.+]] = alloca [8 x i8]
+// CHECK-NEXT: store i64 %b, ptr %[[RET]]
+// CHECK-NEXT: %[[TEMP:.+]] = load <1 x i64>, ptr %[[RET]]
+// CHECK-NEXT: ret <1 x i64> %[[TEMP]]
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]

tests/codegen/transmute-scalar.rs

@@ -41,9 +41,6 @@ pub fn vec_iterator_cast_primitive(vec: Vec<i8>) -> Vec<u8> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| e as u8).collect()
 }
 
@@ -55,9 +52,6 @@ pub fn vec_iterator_cast_wrapper(vec: Vec<u8>) -> Vec<Wrapper<u8>> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| Wrapper(e)).collect()
 }
 
@@ -86,9 +80,6 @@ pub fn vec_iterator_cast_unwrap(vec: Vec<Wrapper<u8>>) -> Vec<u8> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| e.0).collect()
 }
 
@@ -100,9 +91,6 @@ pub fn vec_iterator_cast_aggregate(vec: Vec<[u64; 4]>) -> Vec<Foo> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
     vec.into_iter().map(|e| unsafe { std::mem::transmute(e) }).collect()
 }
 
@@ -114,9 +102,6 @@ pub fn vec_iterator_cast_deaggregate_tra(vec: Vec<Bar>) -> Vec<[u64; 4]> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
 
     // Safety: For the purpose of this test we assume that Bar layout matches [u64; 4].
     // This currently is not guaranteed for repr(Rust) types, but it happens to work here and
@@ -133,9 +118,6 @@ pub fn vec_iterator_cast_deaggregate_fold(vec: Vec<Baz>) -> Vec<[u64; 4]> {
     // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
     // CHECK-NOT: loop
     // CHECK-NOT: call
-    // CHECK: call{{.+}}void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: loop
-    // CHECK-NOT: call
 
     // Safety: For the purpose of this test we assume that Bar layout matches [u64; 4].
     // This currently is not guaranteed for repr(Rust) types, but it happens to work here and
@@ -156,12 +138,7 @@ pub fn vec_iterator_cast_unwrap_drop(vec: Vec<Wrapper<String>>) -> Vec<String> {
     // CHECK-NOT: call
     // CHECK-NOT: %{{.*}} = mul
     // CHECK-NOT: %{{.*}} = udiv
-    // CHECK: call
-    // CHECK-SAME: void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-    // CHECK-NOT: call
-    // CHECK-NOT: %{{.*}} = mul
-    // CHECK-NOT: %{{.*}} = udiv
+    // CHECK: ret void
 
     vec.into_iter().map(|Wrapper(e)| e).collect()
 }
@@ -178,12 +155,6 @@ pub fn vec_iterator_cast_wrap_drop(vec: Vec<String>) -> Vec<Wrapper<String>> {
     // CHECK-NOT: call
     // CHECK-NOT: %{{.*}} = mul
     // CHECK-NOT: %{{.*}} = udiv
-    // CHECK: call
-    // CHECK-SAME: void @llvm.assume(i1 %{{.+}})
-    // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-    // CHECK-NOT: call
-    // CHECK-NOT: %{{.*}} = mul
-    // CHECK-NOT: %{{.*}} = udiv
     // CHECK: ret void
 
     vec.into_iter().map(Wrapper).collect()