From 63421169ce878613349d6a4fa48783d2677af82d Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Tue, 6 Feb 2024 14:32:00 -0500 Subject: [PATCH] Add "algebraic" versions of the fast-math intrinsics --- compiler/rustc_codegen_gcc/src/builder.rs | 25 ++++++++++ compiler/rustc_codegen_llvm/src/builder.rs | 48 +++++++++++++++++-- compiler/rustc_codegen_llvm/src/intrinsic.rs | 4 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + .../rustc_codegen_ssa/src/mir/intrinsic.rs | 32 +++++++++++++ .../rustc_codegen_ssa/src/traits/builder.rs | 5 ++ .../rustc_hir_analysis/src/check/intrinsic.rs | 12 ++++- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 15 +++++- compiler/rustc_span/src/symbol.rs | 5 ++ library/core/src/intrinsics.rs | 40 ++++++++++++++++ 10 files changed, 178 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 42e61b3ccb5ad..5f1e45383765f 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -705,6 +705,31 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { self.frem(lhs, rhs) } + fn fadd_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> { + // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC. + lhs + rhs + } + + fn fsub_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> { + // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC. + lhs - rhs + } + + fn fmul_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> { + // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC. + lhs * rhs + } + + fn fdiv_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> { + // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC. + lhs / rhs + } + + fn frem_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> { + // NOTE: it seems like we cannot enable fast-mode for a single operation in GCC. + self.frem(lhs, rhs) + } + fn checked_binop(&mut self, oop: OverflowOp, typ: Ty<'_>, lhs: Self::Value, rhs: Self::Value) -> (Self::Value, Self::Value) { self.gcc_checked_binop(oop, typ, lhs, rhs) } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7ed27b33dceaa..cfa266720d2a8 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + fn checked_binop( &mut self, oop: OverflowOp, @@ -1327,17 +1367,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fadd_algebraic(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAlgebraicMath(instr); instr } } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fmul_algebraic(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAlgebraicMath(instr); instr } } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index f1a6f7bd8e690..762679308ab8d 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1867,14 +1867,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, + vector_reduce_fadd_algebraic, false, add, 0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, + vector_reduce_fmul_algebraic, false, mul, 1.0 diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4ad44a42738b3..f25bff33026bd 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1619,6 +1619,7 @@ extern "C" { ) -> &'a Value; pub fn LLVMRustSetFastMath(Instr: &Value); + pub fn LLVMRustSetAlgebraicMath(Instr: &Value); // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 8530bf9e2b363..16f25074d3cba 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -249,6 +249,38 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } } + sym::fadd_algebraic + | sym::fsub_algebraic + | sym::fmul_algebraic + | sym::fdiv_algebraic + | sym::frem_algebraic => match float_type_width(arg_tys[0]) { + Some(_width) => match name { + sym::fadd_algebraic => { + bx.fadd_algebraic(args[0].immediate(), args[1].immediate()) + } + sym::fsub_algebraic => { + bx.fsub_algebraic(args[0].immediate(), args[1].immediate()) + } + sym::fmul_algebraic => { + bx.fmul_algebraic(args[0].immediate(), args[1].immediate()) + } + sym::fdiv_algebraic => { + bx.fdiv_algebraic(args[0].immediate(), args[1].immediate()) + } + sym::frem_algebraic => { + bx.frem_algebraic(args[0].immediate(), args[1].immediate()) + } + _ => bug!(), + }, + None => { + bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicFloatType { + span, + name, + ty: arg_tys[0], + }); + return; + } + }, sym::float_to_int_unchecked => { if float_type_width(arg_tys[0]).is_none() { diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 1c5c78e6ca200..86d3d1260c307 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -86,22 +86,27 @@ pub trait BuilderMethods<'a, 'tcx>: fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fadd_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fsub_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fmul_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fdiv_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 2d0d6611444c5..8e5bc2f84e337 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -112,7 +112,12 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir | sym::forget | sym::black_box | sym::variant_count - | sym::ptr_mask => hir::Unsafety::Normal, + | sym::ptr_mask + | sym::fadd_algebraic + | sym::fsub_algebraic + | sym::fmul_algebraic + | sym::fdiv_algebraic + | sym::frem_algebraic => hir::Unsafety::Normal, _ => hir::Unsafety::Unsafe, }; @@ -370,6 +375,11 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => { (1, vec![param(0), param(0)], param(0)) } + sym::fadd_algebraic + | sym::fsub_algebraic + | sym::fmul_algebraic + | sym::fdiv_algebraic + | sym::frem_algebraic => (1, vec![param(0), param(0)], param(0)), sym::float_to_int_unchecked => (2, vec![param(0)], param(1)), sym::assume => (0, vec![tcx.types.bool], Ty::new_unit(tcx)), diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 0df7b7eed11f9..528b088a8d4b6 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -419,8 +419,7 @@ extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C, report_fatal_error("bad MemoryEffects."); } } - -// Enable a fast-math flag +// Enable all fast-math flags // // https://llvm.org/docs/LangRef.html#fast-math-flags extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { @@ -429,6 +428,18 @@ extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { } } +// Enable fast-math flags which do not let operations return poison. +// +// https://llvm.org/docs/LangRef.html#fast-math-flags +extern "C" void LLVMRustSetAlgebraicMath(LLVMValueRef V) { + if (auto I = dyn_cast(unwrap(V))) { + I->setHasAllowReassoc(true); + I->setHasAllowContract(true); + I->setHasAllowReciprocal(true); + I->setHasNoSignedZeros(true); + } +} + extern "C" LLVMValueRef LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, const char *Name, LLVMAtomicOrdering Order) { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index dbfc89c2d496e..0ff49f4dabd20 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -755,8 +755,10 @@ symbols! { f64_nan, fabsf32, fabsf64, + fadd_algebraic, fadd_fast, fake_variadic, + fdiv_algebraic, fdiv_fast, feature, fence, @@ -776,6 +778,7 @@ symbols! { fmaf32, fmaf64, fmt, + fmul_algebraic, fmul_fast, fn_align, fn_delegation, @@ -801,6 +804,7 @@ symbols! { format_unsafe_arg, freeze, freg, + frem_algebraic, frem_fast, from, from_desugaring, @@ -814,6 +818,7 @@ symbols! { from_usize, from_yeet, fs_create_dir, + fsub_algebraic, fsub_fast, fundamental, future, diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 43124535ab5d8..50f4171d055b6 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1882,6 +1882,46 @@ extern "rust-intrinsic" { #[rustc_nounwind] pub fn frem_fast(a: T, b: T) -> T; + /// Float addition that allows optimizations based on algebraic rules. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_nounwind] + #[rustc_safe_intrinsic] + #[cfg(not(bootstrap))] + pub fn fadd_algebraic(a: T, b: T) -> T; + + /// Float subtraction that allows optimizations based on algebraic rules. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_nounwind] + #[rustc_safe_intrinsic] + #[cfg(not(bootstrap))] + pub fn fsub_algebraic(a: T, b: T) -> T; + + /// Float multiplication that allows optimizations based on algebraic rules. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_nounwind] + #[rustc_safe_intrinsic] + #[cfg(not(bootstrap))] + pub fn fmul_algebraic(a: T, b: T) -> T; + + /// Float division that allows optimizations based on algebraic rules. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_nounwind] + #[rustc_safe_intrinsic] + #[cfg(not(bootstrap))] + pub fn fdiv_algebraic(a: T, b: T) -> T; + + /// Float remainder that allows optimizations based on algebraic rules. + /// + /// This intrinsic does not have a stable counterpart. + #[rustc_nounwind] + #[rustc_safe_intrinsic] + #[cfg(not(bootstrap))] + pub fn frem_algebraic(a: T, b: T) -> T; + /// Convert with LLVM’s fptoui/fptosi, which may return undef for values out of range /// () ///