diff --git a/Makefile b/Makefile index 9d1e47714c5..4cc2d52c218 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,9 @@ endif ifeq ($(ARCH), aarch64) test_compilers_engines += cranelift-jit + ifneq (, $(findstring llvm,$(compilers))) + test_compilers_engines += llvm-native + endif endif compilers := $(filter-out ,$(compilers)) diff --git a/lib/compiler-llvm/src/abi/aarch64_systemv.rs b/lib/compiler-llvm/src/abi/aarch64_systemv.rs new file mode 100644 index 00000000000..c4a5fc36ebb --- /dev/null +++ b/lib/compiler-llvm/src/abi/aarch64_systemv.rs @@ -0,0 +1,555 @@ +use crate::abi::Abi; +use crate::translator::intrinsics::{type_to_llvm, Intrinsics}; +use inkwell::{ + attributes::{Attribute, AttributeLoc}, + builder::Builder, + context::Context, + types::{BasicType, FunctionType, StructType}, + values::{BasicValue, BasicValueEnum, CallSiteValue, FunctionValue, IntValue, PointerValue}, + AddressSpace, +}; +use wasmer_compiler::CompileError; +use wasmer_types::{FunctionType as FuncSig, Type}; + +/// Implementation of the [`Abi`] trait for the Aarch64 ABI on Linux. +pub struct Aarch64SystemV {} + +impl Abi for Aarch64SystemV { + // Given a function definition, retrieve the parameter that is the vmctx pointer. + fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { + func_value + .get_nth_param( + if func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some() + { + 1 + } else { + 0 + }, + ) + .unwrap() + .into_pointer_value() + } + + // Given a wasm function type, produce an llvm function declaration. + fn func_type_to_llvm<'ctx>( + &self, + context: &'ctx Context, + intrinsics: &Intrinsics<'ctx>, + sig: &FuncSig, + ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { + let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); + + let param_types = + std::iter::once(Ok(intrinsics.ctx_ptr_ty.as_basic_type_enum())).chain(user_param_types); + + Ok(match sig.results() { + [] => ( + intrinsics + .void_ty + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [_] => { + let single_value = sig.results()[0]; + ( + type_to_llvm(intrinsics, single_value)? + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + [Type::F32, Type::F32] => { + let f32_ty = intrinsics.f32_ty.as_basic_type_enum(); + ( + context + .struct_type(&[f32_ty, f32_ty], false) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + [Type::F64, Type::F64] => { + let f64_ty = intrinsics.f64_ty.as_basic_type_enum(); + ( + context + .struct_type(&[f64_ty, f64_ty], false) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + [Type::F32, Type::F32, Type::F32] => { + let f32_ty = intrinsics.f32_ty.as_basic_type_enum(); + ( + context + .struct_type(&[f32_ty, f32_ty, f32_ty], false) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + [Type::F32, Type::F32, Type::F32, Type::F32] => { + let f32_ty = intrinsics.f32_ty.as_basic_type_enum(); + ( + context + .struct_type(&[f32_ty, f32_ty, f32_ty, f32_ty], false) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + _ => { + let sig_returns_bitwidths = sig + .results() + .iter() + .map(|ty| match ty { + Type::I32 | Type::F32 => 32, + Type::I64 | Type::F64 => 64, + Type::V128 => 128, + Type::ExternRef => unimplemented!("externref in the llvm backend"), + Type::FuncRef => unimplemented!("funcref in the llvm backend"), + }) + .collect::>(); + match sig_returns_bitwidths.as_slice() { + [32, 32] => ( + intrinsics + .i64_ty + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [32, 64] + | [64, 32] + | [64, 64] + | [32, 32, 32] + | [64, 32, 32] + | [32, 32, 64] + | [32, 32, 32, 32] => ( + intrinsics + .i64_ty + .array_type(2) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + _ => { + let basic_types: Vec<_> = sig + .results() + .iter() + .map(|&ty| type_to_llvm(intrinsics, ty)) + .collect::>()?; + + let sret = context + .struct_type(&basic_types, false) + .ptr_type(AddressSpace::Generic); + + let param_types = + std::iter::once(Ok(sret.as_basic_type_enum())).chain(param_types); + + ( + intrinsics + .void_ty + .fn_type(¶m_types.collect::, _>>()?, false), + vec![( + context.create_enum_attribute( + Attribute::get_named_enum_kind_id("sret"), + 0, + ), + AttributeLoc::Param(0), + )], + ) + } + } + } + }) + } + + // Marshall wasm stack values into function parameters. + fn args_to_call<'ctx>( + &self, + alloca_builder: &Builder<'ctx>, + func_sig: &FuncSig, + ctx_ptr: PointerValue<'ctx>, + llvm_fn_ty: &FunctionType<'ctx>, + values: &[BasicValueEnum<'ctx>], + ) -> Vec> { + // If it's an sret, allocate the return space. + let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { + Some( + alloca_builder.build_alloca( + llvm_fn_ty.get_param_types()[0] + .into_pointer_type() + .get_element_type() + .into_struct_type(), + "sret", + ), + ) + } else { + None + }; + + let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); + + if let Some(sret) = sret { + std::iter::once(sret.as_basic_value_enum()) + .chain(values) + .collect() + } else { + values.collect() + } + } + + // Given a CallSite, extract the returned values and return them in a Vec. + fn rets_from_call<'ctx>( + &self, + builder: &Builder<'ctx>, + intrinsics: &Intrinsics<'ctx>, + call_site: CallSiteValue<'ctx>, + func_sig: &FuncSig, + ) -> Vec> { + let split_i64 = |value: IntValue<'ctx>| -> (IntValue<'ctx>, IntValue<'ctx>) { + assert!(value.get_type() == intrinsics.i64_ty); + let low = builder.build_int_truncate(value, intrinsics.i32_ty, ""); + let lshr = + builder.build_right_shift(value, intrinsics.i64_ty.const_int(32, false), false, ""); + let high = builder.build_int_truncate(lshr, intrinsics.i32_ty, ""); + (low, high) + }; + + let casted = |value: BasicValueEnum<'ctx>, ty: Type| -> BasicValueEnum<'ctx> { + match ty { + Type::I32 => { + assert!( + value.get_type() == intrinsics.i32_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f32_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.i32_ty, "") + } + Type::F32 => { + assert!( + value.get_type() == intrinsics.i32_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f32_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.f32_ty, "") + } + Type::I64 => { + assert!( + value.get_type() == intrinsics.i64_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f64_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.i64_ty, "") + } + Type::F64 => { + assert!( + value.get_type() == intrinsics.i64_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f64_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.f64_ty, "") + } + Type::V128 => { + assert!(value.get_type() == intrinsics.i128_ty.as_basic_type_enum()); + value + } + Type::ExternRef => unimplemented!("externref in the llvm backend"), + Type::FuncRef => unimplemented!("funcref in the llvm backend"), + } + }; + + if let Some(basic_value) = call_site.try_as_basic_value().left() { + if func_sig.results().len() > 1 { + if basic_value.get_type() == intrinsics.i64_ty.as_basic_type_enum() { + assert!(func_sig.results().len() == 2); + let value = basic_value.into_int_value(); + let (low, high) = split_i64(value); + let low = casted(low.into(), func_sig.results()[0]); + let high = casted(high.into(), func_sig.results()[1]); + return vec![low, high]; + } + if basic_value.is_struct_value() { + let struct_value = basic_value.into_struct_value(); + return (0..struct_value.get_type().count_fields()) + .map(|i| builder.build_extract_value(struct_value, i, "").unwrap()) + .collect::>(); + } + let array_value = basic_value.into_array_value(); + let low = builder + .build_extract_value(array_value, 0, "") + .unwrap() + .into_int_value(); + let high = builder + .build_extract_value(array_value, 1, "") + .unwrap() + .into_int_value(); + let func_sig_returns_bitwidths = func_sig + .results() + .iter() + .map(|ty| match ty { + Type::I32 | Type::F32 => 32, + Type::I64 | Type::F64 => 64, + Type::V128 => 128, + Type::ExternRef => unimplemented!("externref in the llvm backend"), + Type::FuncRef => unimplemented!("funcref in the llvm backend"), + }) + .collect::>(); + + match func_sig_returns_bitwidths.as_slice() { + [32, 64] => { + let (low, _) = split_i64(low); + let low = casted(low.into(), func_sig.results()[0]); + let high = casted(high.into(), func_sig.results()[1]); + vec![low, high] + } + [64, 32] => { + let (high, _) = split_i64(high); + let low = casted(low.into(), func_sig.results()[0]); + let high = casted(high.into(), func_sig.results()[1]); + vec![low, high] + } + [64, 64] => { + let low = casted(low.into(), func_sig.results()[0]); + let high = casted(high.into(), func_sig.results()[1]); + vec![low, high] + } + [32, 32, 32] => { + let (v1, v2) = split_i64(low); + let (v3, _) = split_i64(high); + let v1 = casted(v1.into(), func_sig.results()[0]); + let v2 = casted(v2.into(), func_sig.results()[1]); + let v3 = casted(v3.into(), func_sig.results()[2]); + vec![v1, v2, v3] + } + [32, 32, 64] => { + let (v1, v2) = split_i64(low); + let v1 = casted(v1.into(), func_sig.results()[0]); + let v2 = casted(v2.into(), func_sig.results()[1]); + let v3 = casted(high.into(), func_sig.results()[2]); + vec![v1, v2, v3] + } + [64, 32, 32] => { + let v1 = casted(low.into(), func_sig.results()[0]); + let (v2, v3) = split_i64(high); + let v2 = casted(v2.into(), func_sig.results()[1]); + let v3 = casted(v3.into(), func_sig.results()[2]); + vec![v1, v2, v3] + } + [32, 32, 32, 32] => { + let (v1, v2) = split_i64(low); + let (v3, v4) = split_i64(high); + let v1 = casted(v1.into(), func_sig.results()[0]); + let v2 = casted(v2.into(), func_sig.results()[1]); + let v3 = casted(v3.into(), func_sig.results()[2]); + let v4 = casted(v4.into(), func_sig.results()[3]); + vec![v1, v2, v3, v4] + } + _ => unreachable!("expected an sret for this type"), + } + } else { + assert!(func_sig.results().len() == 1); + vec![basic_value] + } + } else { + assert!(call_site.count_arguments() > 0); // Either sret or vmctx. + if call_site + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some() + { + let sret = call_site + .try_as_basic_value() + .right() + .unwrap() + .get_operand(0) + .unwrap() + .left() + .unwrap() + .into_pointer_value(); + let struct_value = builder.build_load(sret, "").into_struct_value(); + let mut rets: Vec<_> = Vec::new(); + for i in 0..struct_value.get_type().count_fields() { + let value = builder.build_extract_value(struct_value, i, "").unwrap(); + rets.push(value); + } + assert!(func_sig.results().len() == rets.len()); + rets + } else { + assert!(func_sig.results().is_empty()); + vec![] + } + } + } + + fn is_sret(&self, func_sig: &FuncSig) -> Result { + let func_sig_returns_bitwidths = func_sig + .results() + .iter() + .map(|ty| match ty { + Type::I32 | Type::F32 => Ok(32), + Type::I64 | Type::F64 => Ok(64), + Type::V128 => Ok(128), + ty => Err(CompileError::Codegen(format!( + "is_sret: unimplemented wasmer_types type {:?}", + ty + ))), + }) + .collect::, _>>()?; + + Ok(!matches!(func_sig_returns_bitwidths.as_slice(), + [] + | [_] + | [32, 32] + | [32, 64] + | [64, 32] + | [64, 64] + | [32, 32, 32] + | [32, 32, 64] + | [64, 32, 32] + | [32, 32, 32, 32])) + } + + fn pack_values_for_register_return<'ctx>( + &self, + intrinsics: &Intrinsics<'ctx>, + builder: &Builder<'ctx>, + values: &[BasicValueEnum<'ctx>], + func_type: &FunctionType<'ctx>, + ) -> Result, CompileError> { + let is_32 = |value: BasicValueEnum| { + (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i32_ty) + || (value.is_float_value() + && value.into_float_value().get_type() == intrinsics.f32_ty) + }; + let is_64 = |value: BasicValueEnum| { + (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i64_ty) + || (value.is_float_value() + && value.into_float_value().get_type() == intrinsics.f64_ty) + }; + + let pack_i32s = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| { + assert!(low.get_type() == intrinsics.i32_ty.as_basic_type_enum()); + assert!(high.get_type() == intrinsics.i32_ty.as_basic_type_enum()); + let (low, high) = (low.into_int_value(), high.into_int_value()); + let low = builder.build_int_z_extend(low, intrinsics.i64_ty, ""); + let high = builder.build_int_z_extend(high, intrinsics.i64_ty, ""); + let high = builder.build_left_shift(high, intrinsics.i64_ty.const_int(32, false), ""); + builder.build_or(low, high, "").as_basic_value_enum() + }; + + let to_i64 = |v: BasicValueEnum<'ctx>| { + if v.is_float_value() { + let v = v.into_float_value(); + if v.get_type() == intrinsics.f32_ty { + let v = builder + .build_bitcast(v, intrinsics.i32_ty, "") + .into_int_value(); + let v = builder.build_int_z_extend(v, intrinsics.i64_ty, ""); + v.as_basic_value_enum() + } else { + debug_assert!(v.get_type() == intrinsics.f64_ty); + let v = builder.build_bitcast(v, intrinsics.i64_ty, ""); + v.as_basic_value_enum() + } + } else { + let v = v.into_int_value(); + if v.get_type() == intrinsics.i32_ty { + let v = builder.build_int_z_extend(v, intrinsics.i64_ty, ""); + v.as_basic_value_enum() + } else { + debug_assert!(v.get_type() == intrinsics.i64_ty); + v.as_basic_value_enum() + } + } + }; + + let build_struct = |ty: StructType<'ctx>, values: &[BasicValueEnum<'ctx>]| { + let mut struct_value = ty.get_undef(); + for (i, v) in values.iter().enumerate() { + struct_value = builder + .build_insert_value(struct_value, *v, i as u32, "") + .unwrap() + .into_struct_value(); + } + struct_value.as_basic_value_enum() + }; + + let build_2xi64 = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| { + let low = to_i64(low); + let high = to_i64(high); + let value = intrinsics.i64_ty.array_type(2).get_undef(); + let value = builder.build_insert_value(value, low, 0, "").unwrap(); + let value = builder.build_insert_value(value, high, 1, "").unwrap(); + value.as_basic_value_enum() + }; + + Ok(match *values { + [one_value] => one_value, + [v1, v2] + if v1.is_float_value() + && v2.is_float_value() + && v1.into_float_value().get_type() == v2.into_float_value().get_type() => + { + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1, v2], + ) + } + [v1, v2] if is_32(v1) && is_32(v2) => { + let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + pack_i32s(v1, v2) + } + [v1, v2] => build_2xi64(v1, v2), + [v1, v2, v3] + if is_32(v1) + && is_32(v2) + && is_32(v3) + && v1.is_float_value() + && v2.is_float_value() + && v3.is_float_value() => + { + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1, v2, v3], + ) + } + [v1, v2, v3] if is_32(v1) && is_32(v2) => { + let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + let v1v2_pack = pack_i32s(v1, v2); + build_2xi64(v1v2_pack, v3) + } + [v1, v2, v3] if is_64(v1) && is_32(v2) && is_32(v3) => { + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + let v3 = builder.build_bitcast(v3, intrinsics.i32_ty, ""); + let v2v3_pack = pack_i32s(v2, v3); + build_2xi64(v1, v2v3_pack) + } + [v1, v2, v3, v4] + if is_32(v1) + && is_32(v2) + && is_32(v3) + && is_32(v4) + && v1.is_float_value() + && v2.is_float_value() + && v3.is_float_value() + && v4.is_float_value() => + { + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1, v2, v3, v4], + ) + } + [v1, v2, v3, v4] if is_32(v1) && is_32(v2) && is_32(v3) && is_32(v4) => { + let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + let v1v2_pack = pack_i32s(v1, v2); + let v3 = builder.build_bitcast(v3, intrinsics.i32_ty, ""); + let v4 = builder.build_bitcast(v4, intrinsics.i32_ty, ""); + let v3v4_pack = pack_i32s(v3, v4); + build_2xi64(v1v2_pack, v3v4_pack) + } + _ => { + unreachable!("called to perform register return on struct return or void function") + } + }) + } +} diff --git a/lib/compiler-llvm/src/abi/mod.rs b/lib/compiler-llvm/src/abi/mod.rs new file mode 100644 index 00000000000..2464abfe3c9 --- /dev/null +++ b/lib/compiler-llvm/src/abi/mod.rs @@ -0,0 +1,85 @@ +// LLVM implements part of the ABI lowering internally, but also requires that +// the user pack and unpack values themselves sometimes. This can help the LLVM +// optimizer by exposing operations to the optimizer, but it requires that the +// frontend know exactly what IR to produce in order to get the right ABI. + +#![deny(dead_code, missing_docs)] + +use crate::translator::intrinsics::Intrinsics; +use inkwell::{ + attributes::{Attribute, AttributeLoc}, + builder::Builder, + context::Context, + targets::TargetMachine, + types::FunctionType, + values::{BasicValueEnum, CallSiteValue, FunctionValue, PointerValue}, +}; +use wasmer_compiler::CompileError; +use wasmer_types::FunctionType as FuncSig; + +mod aarch64_systemv; +mod x86_64_systemv; + +use aarch64_systemv::Aarch64SystemV; +use x86_64_systemv::X86_64SystemV; + +pub fn get_abi(target_machine: &TargetMachine) -> Box { + if target_machine + .get_triple() + .as_str() + .to_string_lossy() + .starts_with("aarch64") + { + Box::new(Aarch64SystemV {}) + } else { + Box::new(X86_64SystemV {}) + } +} + +/// We need to produce different LLVM IR for different platforms. (Contrary to +/// popular knowledge LLVM IR is not intended to be portable in that way.) This +/// trait deals with differences between function signatures on different +/// targets. +pub trait Abi { + /// Given a function definition, retrieve the parameter that is the vmctx pointer. + fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx>; + + /// Given a wasm function type, produce an llvm function declaration. + fn func_type_to_llvm<'ctx>( + &self, + context: &'ctx Context, + intrinsics: &Intrinsics<'ctx>, + sig: &FuncSig, + ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError>; + + /// Marshall wasm stack values into function parameters. + fn args_to_call<'ctx>( + &self, + alloca_builder: &Builder<'ctx>, + func_sig: &FuncSig, + ctx_ptr: PointerValue<'ctx>, + llvm_fn_ty: &FunctionType<'ctx>, + values: &[BasicValueEnum<'ctx>], + ) -> Vec>; + + /// Given a CallSite, extract the returned values and return them in a Vec. + fn rets_from_call<'ctx>( + &self, + builder: &Builder<'ctx>, + intrinsics: &Intrinsics<'ctx>, + call_site: CallSiteValue<'ctx>, + func_sig: &FuncSig, + ) -> Vec>; + + /// Whether the llvm equivalent of this wasm function has an `sret` attribute. + fn is_sret(&self, func_sig: &FuncSig) -> Result; + + /// Pack LLVM IR values representing individual wasm values into the return type for the function. + fn pack_values_for_register_return<'ctx>( + &self, + intrinsics: &Intrinsics<'ctx>, + builder: &Builder<'ctx>, + values: &[BasicValueEnum<'ctx>], + func_type: &FunctionType<'ctx>, + ) -> Result, CompileError>; +} diff --git a/lib/compiler-llvm/src/abi/x86_64_systemv.rs b/lib/compiler-llvm/src/abi/x86_64_systemv.rs new file mode 100644 index 00000000000..a6fd67622c7 --- /dev/null +++ b/lib/compiler-llvm/src/abi/x86_64_systemv.rs @@ -0,0 +1,583 @@ +use crate::abi::Abi; +use crate::translator::intrinsics::{type_to_llvm, Intrinsics}; +use inkwell::{ + attributes::{Attribute, AttributeLoc}, + builder::Builder, + context::Context, + types::{BasicType, FunctionType, StructType}, + values::{ + BasicValue, BasicValueEnum, CallSiteValue, FloatValue, FunctionValue, IntValue, + PointerValue, VectorValue, + }, + AddressSpace, +}; +use wasmer_compiler::CompileError; +use wasmer_types::{FunctionType as FuncSig, Type}; + +/// Implementation of the [`Abi`] trait for the AMD64 SystemV ABI. +pub struct X86_64SystemV {} + +impl Abi for X86_64SystemV { + // Given a function definition, retrieve the parameter that is the vmctx pointer. + fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { + func_value + .get_nth_param( + if func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some() + { + 1 + } else { + 0 + }, + ) + .unwrap() + .into_pointer_value() + } + + // Given a wasm function type, produce an llvm function declaration. + fn func_type_to_llvm<'ctx>( + &self, + context: &'ctx Context, + intrinsics: &Intrinsics<'ctx>, + sig: &FuncSig, + ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { + let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); + + let param_types = + std::iter::once(Ok(intrinsics.ctx_ptr_ty.as_basic_type_enum())).chain(user_param_types); + + let sig_returns_bitwidths = sig + .results() + .iter() + .map(|ty| match ty { + Type::I32 | Type::F32 => 32, + Type::I64 | Type::F64 => 64, + Type::V128 => 128, + Type::ExternRef => unimplemented!("externref in the llvm backend"), + Type::FuncRef => unimplemented!("funcref in the llvm backend"), + }) + .collect::>(); + + Ok(match sig_returns_bitwidths.as_slice() { + [] => ( + intrinsics + .void_ty + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [_] => { + let single_value = sig.results()[0]; + ( + type_to_llvm(intrinsics, single_value)? + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + [32, 64] | [64, 32] | [64, 64] => { + let basic_types: Vec<_> = sig + .results() + .iter() + .map(|&ty| type_to_llvm(intrinsics, ty)) + .collect::>()?; + + ( + context + .struct_type(&basic_types, false) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ) + } + [32, 32] if sig.results()[0] == Type::F32 && sig.results()[1] == Type::F32 => ( + intrinsics + .f32_ty + .vec_type(2) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [32, 32] => ( + intrinsics + .i64_ty + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [32, 32, _] if sig.results()[0] == Type::F32 && sig.results()[1] == Type::F32 => ( + context + .struct_type( + &[ + intrinsics.f32_ty.vec_type(2).as_basic_type_enum(), + type_to_llvm(intrinsics, sig.results()[2])?, + ], + false, + ) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [32, 32, _] => ( + context + .struct_type( + &[ + intrinsics.i64_ty.as_basic_type_enum(), + type_to_llvm(intrinsics, sig.results()[2])?, + ], + false, + ) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [64, 32, 32] if sig.results()[1] == Type::F32 && sig.results()[2] == Type::F32 => ( + context + .struct_type( + &[ + type_to_llvm(intrinsics, sig.results()[0])?, + intrinsics.f32_ty.vec_type(2).as_basic_type_enum(), + ], + false, + ) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [64, 32, 32] => ( + context + .struct_type( + &[ + type_to_llvm(intrinsics, sig.results()[0])?, + intrinsics.i64_ty.as_basic_type_enum(), + ], + false, + ) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + [32, 32, 32, 32] => ( + context + .struct_type( + &[ + if sig.results()[0] == Type::F32 && sig.results()[1] == Type::F32 { + intrinsics.f32_ty.vec_type(2).as_basic_type_enum() + } else { + intrinsics.i64_ty.as_basic_type_enum() + }, + if sig.results()[2] == Type::F32 && sig.results()[3] == Type::F32 { + intrinsics.f32_ty.vec_type(2).as_basic_type_enum() + } else { + intrinsics.i64_ty.as_basic_type_enum() + }, + ], + false, + ) + .fn_type(¶m_types.collect::, _>>()?, false), + vec![], + ), + _ => { + let basic_types: Vec<_> = sig + .results() + .iter() + .map(|&ty| type_to_llvm(intrinsics, ty)) + .collect::>()?; + + let sret = context + .struct_type(&basic_types, false) + .ptr_type(AddressSpace::Generic); + + let param_types = std::iter::once(Ok(sret.as_basic_type_enum())).chain(param_types); + + ( + intrinsics + .void_ty + .fn_type(¶m_types.collect::, _>>()?, false), + vec![( + context.create_enum_attribute(Attribute::get_named_enum_kind_id("sret"), 0), + AttributeLoc::Param(0), + )], + ) + } + }) + } + + // Marshall wasm stack values into function parameters. + fn args_to_call<'ctx>( + &self, + alloca_builder: &Builder<'ctx>, + func_sig: &FuncSig, + ctx_ptr: PointerValue<'ctx>, + llvm_fn_ty: &FunctionType<'ctx>, + values: &[BasicValueEnum<'ctx>], + ) -> Vec> { + // If it's an sret, allocate the return space. + let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { + Some( + alloca_builder.build_alloca( + llvm_fn_ty.get_param_types()[0] + .into_pointer_type() + .get_element_type() + .into_struct_type(), + "sret", + ), + ) + } else { + None + }; + + let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); + + if let Some(sret) = sret { + std::iter::once(sret.as_basic_value_enum()) + .chain(values) + .collect() + } else { + values.collect() + } + } + + // Given a CallSite, extract the returned values and return them in a Vec. + fn rets_from_call<'ctx>( + &self, + builder: &Builder<'ctx>, + intrinsics: &Intrinsics<'ctx>, + call_site: CallSiteValue<'ctx>, + func_sig: &FuncSig, + ) -> Vec> { + let split_i64 = |value: IntValue<'ctx>| -> (IntValue<'ctx>, IntValue<'ctx>) { + assert!(value.get_type() == intrinsics.i64_ty); + let low = builder.build_int_truncate(value, intrinsics.i32_ty, ""); + let lshr = + builder.build_right_shift(value, intrinsics.i64_ty.const_int(32, false), false, ""); + let high = builder.build_int_truncate(lshr, intrinsics.i32_ty, ""); + (low, high) + }; + + let f32x2_ty = intrinsics.f32_ty.vec_type(2).as_basic_type_enum(); + let extract_f32x2 = |value: VectorValue<'ctx>| -> (FloatValue<'ctx>, FloatValue<'ctx>) { + assert!(value.get_type() == f32x2_ty.into_vector_type()); + let ret0 = builder + .build_extract_element(value, intrinsics.i32_ty.const_int(0, false), "") + .into_float_value(); + let ret1 = builder + .build_extract_element(value, intrinsics.i32_ty.const_int(1, false), "") + .into_float_value(); + (ret0, ret1) + }; + + let casted = |value: BasicValueEnum<'ctx>, ty: Type| -> BasicValueEnum<'ctx> { + match ty { + Type::I32 => { + assert!( + value.get_type() == intrinsics.i32_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f32_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.i32_ty, "") + } + Type::F32 => { + assert!( + value.get_type() == intrinsics.i32_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f32_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.f32_ty, "") + } + Type::I64 => { + assert!( + value.get_type() == intrinsics.i64_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f64_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.i64_ty, "") + } + Type::F64 => { + assert!( + value.get_type() == intrinsics.i64_ty.as_basic_type_enum() + || value.get_type() == intrinsics.f64_ty.as_basic_type_enum() + ); + builder.build_bitcast(value, intrinsics.f64_ty, "") + } + Type::V128 => { + assert!(value.get_type() == intrinsics.i128_ty.as_basic_type_enum()); + value + } + Type::ExternRef => unimplemented!("externref in the llvm backend"), + Type::FuncRef => unimplemented!("funcref in the llvm backend"), + } + }; + + if let Some(basic_value) = call_site.try_as_basic_value().left() { + if func_sig.results().len() > 1 { + if basic_value.get_type() == intrinsics.i64_ty.as_basic_type_enum() { + assert!(func_sig.results().len() == 2); + let value = basic_value.into_int_value(); + let (low, high) = split_i64(value); + let low = casted(low.into(), func_sig.results()[0]); + let high = casted(high.into(), func_sig.results()[1]); + return vec![low, high]; + } + if basic_value.get_type() == f32x2_ty { + assert!(func_sig.results().len() == 2); + let (ret0, ret1) = extract_f32x2(basic_value.into_vector_value()); + return vec![ret0.into(), ret1.into()]; + } + let struct_value = basic_value.into_struct_value(); + let rets = (0..struct_value.get_type().count_fields()) + .map(|i| builder.build_extract_value(struct_value, i, "").unwrap()) + .collect::>(); + let func_sig_returns_bitwidths = func_sig + .results() + .iter() + .map(|ty| match ty { + Type::I32 | Type::F32 => 32, + Type::I64 | Type::F64 => 64, + Type::V128 => 128, + Type::ExternRef => unimplemented!("externref in the llvm backend"), + Type::FuncRef => unimplemented!("funcref in the llvm backend"), + }) + .collect::>(); + + match func_sig_returns_bitwidths.as_slice() { + [32, 64] | [64, 32] | [64, 64] => { + assert!(func_sig.results().len() == 2); + vec![rets[0], rets[1]] + } + [32, 32, _] + if rets[0].get_type() + == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() => + { + assert!(func_sig.results().len() == 3); + let (rets0, rets1) = extract_f32x2(rets[0].into_vector_value()); + vec![rets0.into(), rets1.into(), rets[1]] + } + [32, 32, _] => { + assert!(func_sig.results().len() == 3); + let (low, high) = split_i64(rets[0].into_int_value()); + let low = casted(low.into(), func_sig.results()[0]); + let high = casted(high.into(), func_sig.results()[1]); + vec![low, high, rets[1]] + } + [64, 32, 32] + if rets[1].get_type() + == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() => + { + assert!(func_sig.results().len() == 3); + let (rets1, rets2) = extract_f32x2(rets[1].into_vector_value()); + vec![rets[0], rets1.into(), rets2.into()] + } + [64, 32, 32] => { + assert!(func_sig.results().len() == 3); + let (rets1, rets2) = split_i64(rets[1].into_int_value()); + let rets1 = casted(rets1.into(), func_sig.results()[1]); + let rets2 = casted(rets2.into(), func_sig.results()[2]); + vec![rets[0], rets1, rets2] + } + [32, 32, 32, 32] => { + assert!(func_sig.results().len() == 4); + let (low0, high0) = if rets[0].get_type() + == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() + { + let (x, y) = extract_f32x2(rets[0].into_vector_value()); + (x.into(), y.into()) + } else { + let (x, y) = split_i64(rets[0].into_int_value()); + (x.into(), y.into()) + }; + let (low1, high1) = if rets[1].get_type() + == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() + { + let (x, y) = extract_f32x2(rets[1].into_vector_value()); + (x.into(), y.into()) + } else { + let (x, y) = split_i64(rets[1].into_int_value()); + (x.into(), y.into()) + }; + let low0 = casted(low0, func_sig.results()[0]); + let high0 = casted(high0, func_sig.results()[1]); + let low1 = casted(low1, func_sig.results()[2]); + let high1 = casted(high1, func_sig.results()[3]); + vec![low0, high0, low1, high1] + } + _ => unreachable!("expected an sret for this type"), + } + } else { + assert!(func_sig.results().len() == 1); + vec![basic_value] + } + } else { + assert!(call_site.count_arguments() > 0); // Either sret or vmctx. + if call_site + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some() + { + let sret = call_site + .try_as_basic_value() + .right() + .unwrap() + .get_operand(0) + .unwrap() + .left() + .unwrap() + .into_pointer_value(); + let struct_value = builder.build_load(sret, "").into_struct_value(); + let mut rets: Vec<_> = Vec::new(); + for i in 0..struct_value.get_type().count_fields() { + let value = builder.build_extract_value(struct_value, i, "").unwrap(); + rets.push(value); + } + assert!(func_sig.results().len() == rets.len()); + rets + } else { + assert!(func_sig.results().is_empty()); + vec![] + } + } + } + + fn is_sret(&self, func_sig: &FuncSig) -> Result { + let func_sig_returns_bitwidths = func_sig + .results() + .iter() + .map(|ty| match ty { + Type::I32 | Type::F32 => Ok(32), + Type::I64 | Type::F64 => Ok(64), + Type::V128 => Ok(128), + ty => Err(CompileError::Codegen(format!( + "is_sret: unimplemented wasmer_types type {:?}", + ty + ))), + }) + .collect::, _>>()?; + + Ok(!matches!(func_sig_returns_bitwidths.as_slice(), + [] + | [_] + | [32, 32] + | [32, 64] + | [64, 32] + | [64, 64] + | [32, 32, 32] + | [32, 32, 64] + | [64, 32, 32] + | [32, 32, 32, 32])) + } + + fn pack_values_for_register_return<'ctx>( + &self, + intrinsics: &Intrinsics<'ctx>, + builder: &Builder<'ctx>, + values: &[BasicValueEnum<'ctx>], + func_type: &FunctionType<'ctx>, + ) -> Result, CompileError> { + let is_32 = |value: BasicValueEnum| { + (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i32_ty) + || (value.is_float_value() + && value.into_float_value().get_type() == intrinsics.f32_ty) + }; + let is_64 = |value: BasicValueEnum| { + (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i64_ty) + || (value.is_float_value() + && value.into_float_value().get_type() == intrinsics.f64_ty) + }; + let is_f32 = |value: BasicValueEnum| { + value.is_float_value() && value.into_float_value().get_type() == intrinsics.f32_ty + }; + + let pack_i32s = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| { + assert!(low.get_type() == intrinsics.i32_ty.as_basic_type_enum()); + assert!(high.get_type() == intrinsics.i32_ty.as_basic_type_enum()); + let (low, high) = (low.into_int_value(), high.into_int_value()); + let low = builder.build_int_z_extend(low, intrinsics.i64_ty, ""); + let high = builder.build_int_z_extend(high, intrinsics.i64_ty, ""); + let high = builder.build_left_shift(high, intrinsics.i64_ty.const_int(32, false), ""); + builder.build_or(low, high, "").as_basic_value_enum() + }; + + let pack_f32s = |first: BasicValueEnum<'ctx>, + second: BasicValueEnum<'ctx>| + -> BasicValueEnum<'ctx> { + assert!(first.get_type() == intrinsics.f32_ty.as_basic_type_enum()); + assert!(second.get_type() == intrinsics.f32_ty.as_basic_type_enum()); + let (first, second) = (first.into_float_value(), second.into_float_value()); + let vec_ty = intrinsics.f32_ty.vec_type(2); + let vec = + builder.build_insert_element(vec_ty.get_undef(), first, intrinsics.i32_zero, ""); + builder + .build_insert_element(vec, second, intrinsics.i32_ty.const_int(1, false), "") + .as_basic_value_enum() + }; + + let build_struct = |ty: StructType<'ctx>, values: &[BasicValueEnum<'ctx>]| { + let mut struct_value = ty.get_undef(); + for (i, v) in values.iter().enumerate() { + struct_value = builder + .build_insert_value(struct_value, *v, i as u32, "") + .unwrap() + .into_struct_value(); + } + struct_value.as_basic_value_enum() + }; + + Ok(match *values { + [one_value] => one_value, + [v1, v2] if is_f32(v1) && is_f32(v2) => pack_f32s(v1, v2), + [v1, v2] if is_32(v1) && is_32(v2) => { + let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + pack_i32s(v1, v2) + } + [v1, v2] => { + assert!(!(is_32(v1) && is_32(v2))); + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1, v2], + ) + } + [v1, v2, v3] if is_f32(v1) && is_f32(v2) => build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[pack_f32s(v1, v2), v3], + ), + [v1, v2, v3] if is_32(v1) && is_32(v2) => { + let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[pack_i32s(v1, v2), v3], + ) + } + [v1, v2, v3] if is_64(v1) && is_f32(v2) && is_f32(v3) => build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1, pack_f32s(v2, v3)], + ), + [v1, v2, v3] if is_64(v1) && is_32(v2) && is_32(v3) => { + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + let v3 = builder.build_bitcast(v3, intrinsics.i32_ty, ""); + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1, pack_i32s(v2, v3)], + ) + } + [v1, v2, v3, v4] if is_32(v1) && is_32(v2) && is_32(v3) && is_32(v4) => { + let v1v2_pack = if is_f32(v1) && is_f32(v2) { + pack_f32s(v1, v2) + } else { + let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); + let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); + pack_i32s(v1, v2) + }; + let v3v4_pack = if is_f32(v3) && is_f32(v4) { + pack_f32s(v3, v4) + } else { + let v3 = builder.build_bitcast(v3, intrinsics.i32_ty, ""); + let v4 = builder.build_bitcast(v4, intrinsics.i32_ty, ""); + pack_i32s(v3, v4) + }; + build_struct( + func_type.get_return_type().unwrap().into_struct_type(), + &[v1v2_pack, v3v4_pack], + ) + } + _ => { + unreachable!("called to perform register return on struct return or void function") + } + }) + } +} diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 2ef80282355..7c1b4f69e2e 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -59,7 +59,7 @@ impl SymbolRegistry for ShortNames { Ok(v) => v, Err(_) => return None, }; - match ty.chars().nth(0).unwrap() { + match ty.chars().next().unwrap() { 'f' => Some(Symbol::LocalFunction(LocalFunctionIndex::from_u32(idx))), 's' => Some(Symbol::Section(SectionIndex::from_u32(idx))), 't' => Some(Symbol::FunctionCallTrampoline(SignatureIndex::from_u32( @@ -267,7 +267,7 @@ impl Compiler for LLVMCompiler { self.config(), memory_styles, &table_styles, - &mut ShortNames {}, + &ShortNames {}, ) }, ) diff --git a/lib/compiler-llvm/src/lib.rs b/lib/compiler-llvm/src/lib.rs index 4b20bb08cc4..5bbbc73098a 100644 --- a/lib/compiler-llvm/src/lib.rs +++ b/lib/compiler-llvm/src/lib.rs @@ -14,6 +14,7 @@ #![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] #![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] +mod abi; mod compiler; mod config; mod object_file; diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index ccc7d1f40a2..d50d5e2e62a 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -1,9 +1,6 @@ +use crate::abi::{get_abi, Abi}; use crate::config::{CompiledKind, LLVM}; use crate::object_file::{load_object_file, CompiledFunction}; -use crate::translator::abi::{ - func_type_to_llvm, get_vmctx_ptr_param, is_sret, pack_values_for_register_return, - rets_from_call, -}; use crate::translator::intrinsics::{type_to_llvm, type_to_llvm_ptr, Intrinsics}; use inkwell::{ attributes::{Attribute, AttributeLoc}, @@ -23,15 +20,18 @@ use wasmer_types::{FunctionType, LocalFunctionIndex}; pub struct FuncTrampoline { ctx: Context, target_machine: TargetMachine, + abi: Box, } const FUNCTION_SECTION: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars impl FuncTrampoline { pub fn new(target_machine: TargetMachine) -> Self { + let abi = get_abi(&target_machine); Self { ctx: Context::create(), target_machine, + abi, } } @@ -50,7 +50,7 @@ impl FuncTrampoline { module.set_data_layout(&target_machine.get_target_data().get_data_layout()); let intrinsics = Intrinsics::declare(&module, &self.ctx); - let (callee_ty, callee_attrs) = func_type_to_llvm(&self.ctx, &intrinsics, ty)?; + let (callee_ty, callee_attrs) = self.abi.func_type_to_llvm(&self.ctx, &intrinsics, ty)?; let trampoline_ty = intrinsics.void_ty.fn_type( &[ intrinsics.ctx_ptr_ty.as_basic_type_enum(), // vmctx ptr @@ -72,7 +72,7 @@ impl FuncTrampoline { trampoline_func .as_global_value() .set_dll_storage_class(DLLStorageClass::Export); - generate_trampoline(trampoline_func, ty, &callee_attrs, &self.ctx, &intrinsics)?; + self.generate_trampoline(trampoline_func, ty, &callee_attrs, &self.ctx, &intrinsics)?; if let Some(ref callbacks) = config.callbacks { callbacks.preopt_ir(&function, &module); @@ -180,7 +180,8 @@ impl FuncTrampoline { module.set_data_layout(&target_machine.get_target_data().get_data_layout()); let intrinsics = Intrinsics::declare(&module, &self.ctx); - let (trampoline_ty, trampoline_attrs) = func_type_to_llvm(&self.ctx, &intrinsics, ty)?; + let (trampoline_ty, trampoline_attrs) = + self.abi.func_type_to_llvm(&self.ctx, &intrinsics, ty)?; let trampoline_func = module.add_function(name, trampoline_ty, Some(Linkage::External)); for (attr, attr_loc) in trampoline_attrs { trampoline_func.add_attribute(attr_loc, attr); @@ -194,7 +195,7 @@ impl FuncTrampoline { trampoline_func .as_global_value() .set_dll_storage_class(DLLStorageClass::Export); - generate_dynamic_trampoline(trampoline_func, ty, &self.ctx, &intrinsics)?; + self.generate_dynamic_trampoline(trampoline_func, ty, &self.ctx, &intrinsics)?; if let Some(ref callbacks) = config.callbacks { callbacks.preopt_ir(&function, &module); @@ -286,225 +287,223 @@ impl FuncTrampoline { unwind_info: compiled_function.body.unwind_info, }) } -} -fn generate_trampoline<'ctx>( - trampoline_func: FunctionValue, - func_sig: &FunctionType, - func_attrs: &[(Attribute, AttributeLoc)], - context: &'ctx Context, - intrinsics: &Intrinsics<'ctx>, -) -> Result<(), CompileError> { - let entry_block = context.append_basic_block(trampoline_func, "entry"); - let builder = context.create_builder(); - builder.position_at_end(entry_block); - - /* - // TODO: remove debugging - builder.build_call( - intrinsics.debug_trap, - &[], - ""); - */ - - let (callee_vmctx_ptr, func_ptr, args_rets_ptr) = match *trampoline_func.get_params().as_slice() - { - [callee_vmctx_ptr, func_ptr, args_rets_ptr] => ( - callee_vmctx_ptr, - func_ptr.into_pointer_value(), - args_rets_ptr.into_pointer_value(), - ), - _ => { - return Err(CompileError::Codegen( - "trampoline function unimplemented".to_string(), - )) + fn generate_trampoline<'ctx>( + &self, + trampoline_func: FunctionValue, + func_sig: &FunctionType, + func_attrs: &[(Attribute, AttributeLoc)], + context: &'ctx Context, + intrinsics: &Intrinsics<'ctx>, + ) -> Result<(), CompileError> { + let entry_block = context.append_basic_block(trampoline_func, "entry"); + let builder = context.create_builder(); + builder.position_at_end(entry_block); + + let (callee_vmctx_ptr, func_ptr, args_rets_ptr) = + match *trampoline_func.get_params().as_slice() { + [callee_vmctx_ptr, func_ptr, args_rets_ptr] => ( + callee_vmctx_ptr, + func_ptr.into_pointer_value(), + args_rets_ptr.into_pointer_value(), + ), + _ => { + return Err(CompileError::Codegen( + "trampoline function unimplemented".to_string(), + )) + } + }; + + let mut args_vec = Vec::with_capacity(func_sig.params().len() + 1); + + if self.abi.is_sret(func_sig)? { + let basic_types: Vec<_> = func_sig + .results() + .iter() + .map(|&ty| type_to_llvm(intrinsics, ty)) + .collect::>()?; + + let sret_ty = context.struct_type(&basic_types, false); + args_vec.push(builder.build_alloca(sret_ty, "sret").into()); } - }; - let mut args_vec = Vec::with_capacity(func_sig.params().len() + 1); + args_vec.push(callee_vmctx_ptr); - if is_sret(func_sig)? { - let basic_types: Vec<_> = func_sig - .results() - .iter() - .map(|&ty| type_to_llvm(intrinsics, ty)) - .collect::>()?; + for (i, param_ty) in func_sig.params().iter().enumerate() { + let index = intrinsics.i32_ty.const_int(i as _, false); + let item_pointer = + unsafe { builder.build_in_bounds_gep(args_rets_ptr, &[index], "arg_ptr") }; - let sret_ty = context.struct_type(&basic_types, false); - args_vec.push(builder.build_alloca(sret_ty, "sret").into()); - } + let casted_pointer_type = type_to_llvm_ptr(intrinsics, *param_ty)?; - args_vec.push(callee_vmctx_ptr); + let typed_item_pointer = + builder.build_pointer_cast(item_pointer, casted_pointer_type, "typed_arg_pointer"); - for (i, param_ty) in func_sig.params().iter().enumerate() { - let index = intrinsics.i32_ty.const_int(i as _, false); - let item_pointer = - unsafe { builder.build_in_bounds_gep(args_rets_ptr, &[index], "arg_ptr") }; + let arg = builder.build_load(typed_item_pointer, "arg"); + args_vec.push(arg); + } - let casted_pointer_type = type_to_llvm_ptr(intrinsics, *param_ty)?; + let call_site = builder.build_call(func_ptr, &args_vec, "call"); + for (attr, attr_loc) in func_attrs { + call_site.add_attribute(*attr_loc, *attr); + } - let typed_item_pointer = - builder.build_pointer_cast(item_pointer, casted_pointer_type, "typed_arg_pointer"); + let rets = self + .abi + .rets_from_call(&builder, intrinsics, call_site, func_sig); + let mut idx = 0; + rets.iter().for_each(|v| { + let ptr = unsafe { + builder.build_gep( + args_rets_ptr, + &[intrinsics.i32_ty.const_int(idx, false)], + "", + ) + }; + let ptr = + builder.build_pointer_cast(ptr, v.get_type().ptr_type(AddressSpace::Generic), ""); + builder.build_store(ptr, *v); + if v.get_type() == intrinsics.i128_ty.as_basic_type_enum() { + idx += 1; + } + idx += 1; + }); - let arg = builder.build_load(typed_item_pointer, "arg"); - args_vec.push(arg); + builder.build_return(None); + Ok(()) } - let call_site = builder.build_call(func_ptr, &args_vec, "call"); - for (attr, attr_loc) in func_attrs { - call_site.add_attribute(*attr_loc, *attr); - } + fn generate_dynamic_trampoline<'ctx>( + &self, + trampoline_func: FunctionValue, + func_sig: &FunctionType, + context: &'ctx Context, + intrinsics: &Intrinsics<'ctx>, + ) -> Result<(), CompileError> { + let entry_block = context.append_basic_block(trampoline_func, "entry"); + let builder = context.create_builder(); + builder.position_at_end(entry_block); + + // Allocate stack space for the params and results. + let values = builder.build_alloca( + intrinsics.i128_ty.array_type(cmp::max( + func_sig.params().len().try_into().unwrap(), + func_sig.results().len().try_into().unwrap(), + )), + "", + ); - let rets = rets_from_call(&builder, intrinsics, call_site, func_sig); - let mut idx = 0; - rets.iter().for_each(|v| { - let ptr = unsafe { - builder.build_gep( - args_rets_ptr, - &[intrinsics.i32_ty.const_int(idx, false)], - "", - ) - }; - let ptr = builder.build_pointer_cast(ptr, v.get_type().ptr_type(AddressSpace::Generic), ""); - builder.build_store(ptr, *v); - if v.get_type() == intrinsics.i128_ty.as_basic_type_enum() { - idx += 1; + // Copy params to 'values'. + let first_user_param = if self.abi.is_sret(func_sig)? { 2 } else { 1 }; + for i in 0..func_sig.params().len() { + let ptr = unsafe { + builder.build_in_bounds_gep( + values, + &[ + intrinsics.i32_zero, + intrinsics.i32_ty.const_int(i.try_into().unwrap(), false), + ], + "", + ) + }; + let ptr = builder + .build_bitcast(ptr, type_to_llvm_ptr(intrinsics, func_sig.params()[i])?, "") + .into_pointer_value(); + builder.build_store( + ptr, + trampoline_func + .get_nth_param(i as u32 + first_user_param) + .unwrap(), + ); } - idx += 1; - }); - - builder.build_return(None); - Ok(()) -} -fn generate_dynamic_trampoline<'ctx>( - trampoline_func: FunctionValue, - func_sig: &FunctionType, - context: &'ctx Context, - intrinsics: &Intrinsics<'ctx>, -) -> Result<(), CompileError> { - let entry_block = context.append_basic_block(trampoline_func, "entry"); - let builder = context.create_builder(); - builder.position_at_end(entry_block); - - // Allocate stack space for the params and results. - let values = builder.build_alloca( - intrinsics.i128_ty.array_type(cmp::max( - func_sig.params().len().try_into().unwrap(), - func_sig.results().len().try_into().unwrap(), - )), - "", - ); - - // Copy params to 'values'. - let first_user_param = if is_sret(func_sig)? { 2 } else { 1 }; - for i in 0..func_sig.params().len() { - let ptr = unsafe { - builder.build_in_bounds_gep( - values, + let callee_ty = intrinsics + .void_ty + .fn_type( &[ - intrinsics.i32_zero, - intrinsics.i32_ty.const_int(i.try_into().unwrap(), false), + intrinsics.ctx_ptr_ty.as_basic_type_enum(), // vmctx ptr + intrinsics.i128_ptr_ty.as_basic_type_enum(), // in/out values ptr ], + false, + ) + .ptr_type(AddressSpace::Generic); + let vmctx = self.abi.get_vmctx_ptr_param(&trampoline_func); + let callee = builder + .build_load( + builder + .build_bitcast(vmctx, callee_ty.ptr_type(AddressSpace::Generic), "") + .into_pointer_value(), "", ) - }; - let ptr = builder - .build_bitcast(ptr, type_to_llvm_ptr(intrinsics, func_sig.params()[i])?, "") .into_pointer_value(); - builder.build_store( - ptr, - trampoline_func - .get_nth_param(i as u32 + first_user_param) - .unwrap(), - ); - } - let callee_ty = intrinsics - .void_ty - .fn_type( + let values_ptr = builder.build_pointer_cast(values, intrinsics.i128_ptr_ty, ""); + builder.build_call( + callee, &[ - intrinsics.ctx_ptr_ty.as_basic_type_enum(), // vmctx ptr - intrinsics.i128_ptr_ty.as_basic_type_enum(), // in/out values ptr + vmctx.as_basic_value_enum(), + values_ptr.as_basic_value_enum(), ], - false, - ) - .ptr_type(AddressSpace::Generic); - let vmctx = get_vmctx_ptr_param(&trampoline_func); - let callee = builder - .build_load( - builder - .build_bitcast(vmctx, callee_ty.ptr_type(AddressSpace::Generic), "") - .into_pointer_value(), "", - ) - .into_pointer_value(); - - let values_ptr = builder.build_pointer_cast(values, intrinsics.i128_ptr_ty, ""); - builder.build_call( - callee, - &[ - vmctx.as_basic_value_enum(), - values_ptr.as_basic_value_enum(), - ], - "", - ); - - if func_sig.results().is_empty() { - builder.build_return(None); - } else { - let results = func_sig - .results() - .iter() - .enumerate() - .map(|(idx, ty)| { - let ptr = unsafe { - builder.build_gep( - values, - &[ - intrinsics.i32_ty.const_zero(), - intrinsics.i32_ty.const_int(idx.try_into().unwrap(), false), - ], - "", - ) - }; - let ptr = builder.build_pointer_cast(ptr, type_to_llvm_ptr(intrinsics, *ty)?, ""); - Ok(builder.build_load(ptr, "")) - }) - .collect::, CompileError>>()?; - - if is_sret(func_sig)? { - let sret = trampoline_func - .get_first_param() - .unwrap() - .into_pointer_value(); - let mut struct_value = sret - .get_type() - .get_element_type() - .into_struct_type() - .get_undef(); - for (idx, value) in results.iter().enumerate() { - let value = builder.build_bitcast( - *value, - type_to_llvm(&intrinsics, func_sig.results()[idx])?, - "", - ); - struct_value = builder - .build_insert_value(struct_value, value, idx as u32, "") - .unwrap() - .into_struct_value(); - } - builder.build_store(sret, struct_value); + ); + + if func_sig.results().is_empty() { builder.build_return(None); } else { - builder.build_return(Some(&pack_values_for_register_return( - &intrinsics, - &builder, - &results.as_slice(), - &trampoline_func.get_type(), - )?)); + let results = func_sig + .results() + .iter() + .enumerate() + .map(|(idx, ty)| { + let ptr = unsafe { + builder.build_gep( + values, + &[ + intrinsics.i32_ty.const_zero(), + intrinsics.i32_ty.const_int(idx.try_into().unwrap(), false), + ], + "", + ) + }; + let ptr = + builder.build_pointer_cast(ptr, type_to_llvm_ptr(intrinsics, *ty)?, ""); + Ok(builder.build_load(ptr, "")) + }) + .collect::, CompileError>>()?; + + if self.abi.is_sret(func_sig)? { + let sret = trampoline_func + .get_first_param() + .unwrap() + .into_pointer_value(); + let mut struct_value = sret + .get_type() + .get_element_type() + .into_struct_type() + .get_undef(); + for (idx, value) in results.iter().enumerate() { + let value = builder.build_bitcast( + *value, + type_to_llvm(&intrinsics, func_sig.results()[idx])?, + "", + ); + struct_value = builder + .build_insert_value(struct_value, value, idx as u32, "") + .unwrap() + .into_struct_value(); + } + builder.build_store(sret, struct_value); + builder.build_return(None); + } else { + builder.build_return(Some(&self.abi.pack_values_for_register_return( + &intrinsics, + &builder, + &results.as_slice(), + &trampoline_func.get_type(), + )?)); + } } - } - Ok(()) + Ok(()) + } } diff --git a/lib/compiler-llvm/src/translator/abi.rs b/lib/compiler-llvm/src/translator/abi.rs deleted file mode 100644 index 6e26a026b68..00000000000 --- a/lib/compiler-llvm/src/translator/abi.rs +++ /dev/null @@ -1,577 +0,0 @@ -// LLVM implements part of the ABI lowering internally, but also requires that -// the user pack and unpack values themselves sometimes. This can help the LLVM -// optimizer by exposing operations to the optimizer, but it requires that the -// frontend know exactly what IR to produce in order to get the right ABI. -// -// So far, this is an implementation of the SysV AMD64 ABI. - -#![deny(dead_code, missing_docs)] - -use crate::translator::intrinsics::{type_to_llvm, Intrinsics}; -use inkwell::{ - attributes::{Attribute, AttributeLoc}, - builder::Builder, - context::Context, - types::{BasicType, FunctionType, StructType}, - values::{ - BasicValue, BasicValueEnum, CallSiteValue, FloatValue, FunctionValue, IntValue, - PointerValue, VectorValue, - }, - AddressSpace, -}; -use wasmer_compiler::CompileError; -use wasmer_types::{FunctionType as FuncSig, Type}; - -// Given a function definition, retrieve the parameter that is the vmctx pointer. -pub fn get_vmctx_ptr_param<'ctx>(func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { - func_value - .get_nth_param( - if func_value - .get_enum_attribute( - AttributeLoc::Param(0), - Attribute::get_named_enum_kind_id("sret"), - ) - .is_some() - { - 1 - } else { - 0 - }, - ) - .unwrap() - .into_pointer_value() -} - -// Given a wasm function type, produce an llvm function declaration. -pub fn func_type_to_llvm<'ctx>( - context: &'ctx Context, - intrinsics: &Intrinsics<'ctx>, - sig: &FuncSig, -) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { - let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); - - let param_types = - std::iter::once(Ok(intrinsics.ctx_ptr_ty.as_basic_type_enum())).chain(user_param_types); - - let sig_returns_bitwidths = sig - .results() - .iter() - .map(|ty| match ty { - Type::I32 | Type::F32 => 32, - Type::I64 | Type::F64 => 64, - Type::V128 => 128, - Type::ExternRef => unimplemented!("externref in the llvm backend"), - Type::FuncRef => unimplemented!("funcref in the llvm backend"), - }) - .collect::>(); - - Ok(match sig_returns_bitwidths.as_slice() { - [] => ( - intrinsics - .void_ty - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [_] => { - let single_value = sig.results()[0]; - ( - type_to_llvm(intrinsics, single_value)? - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ) - } - [32, 64] | [64, 32] | [64, 64] => { - let basic_types: Vec<_> = sig - .results() - .iter() - .map(|&ty| type_to_llvm(intrinsics, ty)) - .collect::>()?; - - ( - context - .struct_type(&basic_types, false) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ) - } - [32, 32] if sig.results()[0] == Type::F32 && sig.results()[1] == Type::F32 => ( - intrinsics - .f32_ty - .vec_type(2) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [32, 32] => ( - intrinsics - .i64_ty - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [32, 32, _] if sig.results()[0] == Type::F32 && sig.results()[1] == Type::F32 => ( - context - .struct_type( - &[ - intrinsics.f32_ty.vec_type(2).as_basic_type_enum(), - type_to_llvm(intrinsics, sig.results()[2])?, - ], - false, - ) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [32, 32, _] => ( - context - .struct_type( - &[ - intrinsics.i64_ty.as_basic_type_enum(), - type_to_llvm(intrinsics, sig.results()[2])?, - ], - false, - ) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [64, 32, 32] if sig.results()[1] == Type::F32 && sig.results()[2] == Type::F32 => ( - context - .struct_type( - &[ - type_to_llvm(intrinsics, sig.results()[0])?, - intrinsics.f32_ty.vec_type(2).as_basic_type_enum(), - ], - false, - ) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [64, 32, 32] => ( - context - .struct_type( - &[ - type_to_llvm(intrinsics, sig.results()[0])?, - intrinsics.i64_ty.as_basic_type_enum(), - ], - false, - ) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - [32, 32, 32, 32] => ( - context - .struct_type( - &[ - if sig.results()[0] == Type::F32 && sig.results()[1] == Type::F32 { - intrinsics.f32_ty.vec_type(2).as_basic_type_enum() - } else { - intrinsics.i64_ty.as_basic_type_enum() - }, - if sig.results()[2] == Type::F32 && sig.results()[3] == Type::F32 { - intrinsics.f32_ty.vec_type(2).as_basic_type_enum() - } else { - intrinsics.i64_ty.as_basic_type_enum() - }, - ], - false, - ) - .fn_type(¶m_types.collect::, _>>()?, false), - vec![], - ), - _ => { - let basic_types: Vec<_> = sig - .results() - .iter() - .map(|&ty| type_to_llvm(intrinsics, ty)) - .collect::>()?; - - let sret = context - .struct_type(&basic_types, false) - .ptr_type(AddressSpace::Generic); - - let param_types = std::iter::once(Ok(sret.as_basic_type_enum())).chain(param_types); - - ( - intrinsics - .void_ty - .fn_type(¶m_types.collect::, _>>()?, false), - vec![( - context.create_enum_attribute(Attribute::get_named_enum_kind_id("sret"), 0), - AttributeLoc::Param(0), - )], - ) - } - }) -} - -// Marshall wasm stack values into function parameters. -pub fn args_to_call<'ctx>( - alloca_builder: &Builder<'ctx>, - func_sig: &FuncSig, - ctx_ptr: PointerValue<'ctx>, - llvm_fn_ty: &FunctionType<'ctx>, - values: &[BasicValueEnum<'ctx>], -) -> Vec> { - // If it's an sret, allocate the return space. - let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { - Some( - alloca_builder.build_alloca( - llvm_fn_ty.get_param_types()[0] - .into_pointer_type() - .get_element_type() - .into_struct_type(), - "sret", - ), - ) - } else { - None - }; - - let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); - - if let Some(sret) = sret { - std::iter::once(sret.as_basic_value_enum()) - .chain(values) - .collect() - } else { - values.collect() - } -} - -// Given a CallSite, extract the returned values and return them in a Vec. -pub fn rets_from_call<'ctx>( - builder: &Builder<'ctx>, - intrinsics: &Intrinsics<'ctx>, - call_site: CallSiteValue<'ctx>, - func_sig: &FuncSig, -) -> Vec> { - let split_i64 = |value: IntValue<'ctx>| -> (IntValue<'ctx>, IntValue<'ctx>) { - assert!(value.get_type() == intrinsics.i64_ty); - let low = builder.build_int_truncate(value, intrinsics.i32_ty, ""); - let lshr = - builder.build_right_shift(value, intrinsics.i64_ty.const_int(32, false), false, ""); - let high = builder.build_int_truncate(lshr, intrinsics.i32_ty, ""); - (low, high) - }; - - let f32x2_ty = intrinsics.f32_ty.vec_type(2).as_basic_type_enum(); - let extract_f32x2 = |value: VectorValue<'ctx>| -> (FloatValue<'ctx>, FloatValue<'ctx>) { - assert!(value.get_type() == f32x2_ty.into_vector_type()); - let ret0 = builder - .build_extract_element(value, intrinsics.i32_ty.const_int(0, false), "") - .into_float_value(); - let ret1 = builder - .build_extract_element(value, intrinsics.i32_ty.const_int(1, false), "") - .into_float_value(); - (ret0, ret1) - }; - - let casted = |value: BasicValueEnum<'ctx>, ty: Type| -> BasicValueEnum<'ctx> { - match ty { - Type::I32 => { - assert!( - value.get_type() == intrinsics.i32_ty.as_basic_type_enum() - || value.get_type() == intrinsics.f32_ty.as_basic_type_enum() - ); - builder.build_bitcast(value, intrinsics.i32_ty, "") - } - Type::F32 => { - assert!( - value.get_type() == intrinsics.i32_ty.as_basic_type_enum() - || value.get_type() == intrinsics.f32_ty.as_basic_type_enum() - ); - builder.build_bitcast(value, intrinsics.f32_ty, "") - } - Type::I64 => { - assert!( - value.get_type() == intrinsics.i64_ty.as_basic_type_enum() - || value.get_type() == intrinsics.f64_ty.as_basic_type_enum() - ); - builder.build_bitcast(value, intrinsics.i64_ty, "") - } - Type::F64 => { - assert!( - value.get_type() == intrinsics.i64_ty.as_basic_type_enum() - || value.get_type() == intrinsics.f64_ty.as_basic_type_enum() - ); - builder.build_bitcast(value, intrinsics.f64_ty, "") - } - Type::V128 => { - assert!(value.get_type() == intrinsics.i128_ty.as_basic_type_enum()); - value - } - Type::ExternRef => unimplemented!("externref in the llvm backend"), - Type::FuncRef => unimplemented!("funcref in the llvm backend"), - } - }; - - if let Some(basic_value) = call_site.try_as_basic_value().left() { - if func_sig.results().len() > 1 { - if basic_value.get_type() == intrinsics.i64_ty.as_basic_type_enum() { - assert!(func_sig.results().len() == 2); - let value = basic_value.into_int_value(); - let (low, high) = split_i64(value); - let low = casted(low.into(), func_sig.results()[0]); - let high = casted(high.into(), func_sig.results()[1]); - return vec![low, high]; - } - if basic_value.get_type() == f32x2_ty { - assert!(func_sig.results().len() == 2); - let (ret0, ret1) = extract_f32x2(basic_value.into_vector_value()); - return vec![ret0.into(), ret1.into()]; - } - let struct_value = basic_value.into_struct_value(); - let rets = (0..struct_value.get_type().count_fields()) - .map(|i| builder.build_extract_value(struct_value, i, "").unwrap()) - .collect::>(); - let func_sig_returns_bitwidths = func_sig - .results() - .iter() - .map(|ty| match ty { - Type::I32 | Type::F32 => 32, - Type::I64 | Type::F64 => 64, - Type::V128 => 128, - Type::ExternRef => unimplemented!("externref in the llvm backend"), - Type::FuncRef => unimplemented!("funcref in the llvm backend"), - }) - .collect::>(); - - match func_sig_returns_bitwidths.as_slice() { - [32, 64] | [64, 32] | [64, 64] => { - assert!(func_sig.results().len() == 2); - vec![rets[0], rets[1]] - } - [32, 32, _] - if rets[0].get_type() == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() => - { - assert!(func_sig.results().len() == 3); - let (rets0, rets1) = extract_f32x2(rets[0].into_vector_value()); - vec![rets0.into(), rets1.into(), rets[1]] - } - [32, 32, _] => { - assert!(func_sig.results().len() == 3); - let (low, high) = split_i64(rets[0].into_int_value()); - let low = casted(low.into(), func_sig.results()[0]); - let high = casted(high.into(), func_sig.results()[1]); - vec![low, high, rets[1]] - } - [64, 32, 32] - if rets[1].get_type() == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() => - { - assert!(func_sig.results().len() == 3); - let (rets1, rets2) = extract_f32x2(rets[1].into_vector_value()); - vec![rets[0], rets1.into(), rets2.into()] - } - [64, 32, 32] => { - assert!(func_sig.results().len() == 3); - let (rets1, rets2) = split_i64(rets[1].into_int_value()); - let rets1 = casted(rets1.into(), func_sig.results()[1]); - let rets2 = casted(rets2.into(), func_sig.results()[2]); - vec![rets[0], rets1, rets2] - } - [32, 32, 32, 32] => { - assert!(func_sig.results().len() == 4); - let (low0, high0) = if rets[0].get_type() - == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() - { - let (x, y) = extract_f32x2(rets[0].into_vector_value()); - (x.into(), y.into()) - } else { - let (x, y) = split_i64(rets[0].into_int_value()); - (x.into(), y.into()) - }; - let (low1, high1) = if rets[1].get_type() - == intrinsics.f32_ty.vec_type(2).as_basic_type_enum() - { - let (x, y) = extract_f32x2(rets[1].into_vector_value()); - (x.into(), y.into()) - } else { - let (x, y) = split_i64(rets[1].into_int_value()); - (x.into(), y.into()) - }; - let low0 = casted(low0, func_sig.results()[0]); - let high0 = casted(high0, func_sig.results()[1]); - let low1 = casted(low1, func_sig.results()[2]); - let high1 = casted(high1, func_sig.results()[3]); - vec![low0, high0, low1, high1] - } - _ => unreachable!("expected an sret for this type"), - } - } else { - assert!(func_sig.results().len() == 1); - vec![basic_value] - } - } else { - assert!(call_site.count_arguments() > 0); // Either sret or vmctx. - if call_site - .get_enum_attribute( - AttributeLoc::Param(0), - Attribute::get_named_enum_kind_id("sret"), - ) - .is_some() - { - let sret = call_site - .try_as_basic_value() - .right() - .unwrap() - .get_operand(0) - .unwrap() - .left() - .unwrap() - .into_pointer_value(); - let struct_value = builder.build_load(sret, "").into_struct_value(); - let mut rets: Vec<_> = Vec::new(); - for i in 0..struct_value.get_type().count_fields() { - let value = builder.build_extract_value(struct_value, i, "").unwrap(); - rets.push(value); - } - assert!(func_sig.results().len() == rets.len()); - rets - } else { - assert!(func_sig.results().is_empty()); - vec![] - } - } -} - -pub fn is_sret(func_sig: &FuncSig) -> Result { - let func_sig_returns_bitwidths = func_sig - .results() - .iter() - .map(|ty| match ty { - Type::I32 | Type::F32 => Ok(32), - Type::I64 | Type::F64 => Ok(64), - Type::V128 => Ok(128), - ty => Err(CompileError::Codegen(format!( - "is_sret: unimplemented wasmer_types type {:?}", - ty - ))), - }) - .collect::, _>>()?; - - Ok(match func_sig_returns_bitwidths.as_slice() { - [] - | [_] - | [32, 64] - | [64, 32] - | [64, 64] - | [32, 32] - | [32, 32, 32] - | [32, 32, 64] - | [64, 32, 32] - | [32, 32, 32, 32] => false, - _ => true, - }) -} - -pub fn pack_values_for_register_return<'ctx>( - intrinsics: &Intrinsics<'ctx>, - builder: &Builder<'ctx>, - values: &[BasicValueEnum<'ctx>], - func_type: &FunctionType<'ctx>, -) -> Result, CompileError> { - let is_32 = |value: BasicValueEnum| { - (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i32_ty) - || (value.is_float_value() && value.into_float_value().get_type() == intrinsics.f32_ty) - }; - let is_64 = |value: BasicValueEnum| { - (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i64_ty) - || (value.is_float_value() && value.into_float_value().get_type() == intrinsics.f64_ty) - }; - let is_f32 = |value: BasicValueEnum| { - value.is_float_value() && value.into_float_value().get_type() == intrinsics.f32_ty - }; - - let pack_i32s = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| { - assert!(low.get_type() == intrinsics.i32_ty.as_basic_type_enum()); - assert!(high.get_type() == intrinsics.i32_ty.as_basic_type_enum()); - let (low, high) = (low.into_int_value(), high.into_int_value()); - let low = builder.build_int_z_extend(low, intrinsics.i64_ty, ""); - let high = builder.build_int_z_extend(high, intrinsics.i64_ty, ""); - let high = builder.build_left_shift(high, intrinsics.i64_ty.const_int(32, false), ""); - builder.build_or(low, high, "").as_basic_value_enum() - }; - - let pack_f32s = |first: BasicValueEnum<'ctx>, - second: BasicValueEnum<'ctx>| - -> BasicValueEnum<'ctx> { - assert!(first.get_type() == intrinsics.f32_ty.as_basic_type_enum()); - assert!(second.get_type() == intrinsics.f32_ty.as_basic_type_enum()); - let (first, second) = (first.into_float_value(), second.into_float_value()); - let vec_ty = intrinsics.f32_ty.vec_type(2); - let vec = builder.build_insert_element(vec_ty.get_undef(), first, intrinsics.i32_zero, ""); - builder - .build_insert_element(vec, second, intrinsics.i32_ty.const_int(1, false), "") - .as_basic_value_enum() - }; - - let build_struct = |ty: StructType<'ctx>, values: &[BasicValueEnum<'ctx>]| { - let mut struct_value = ty.get_undef(); - for (i, v) in values.iter().enumerate() { - struct_value = builder - .build_insert_value(struct_value, *v, i as u32, "") - .unwrap() - .into_struct_value(); - } - struct_value.as_basic_value_enum() - }; - - Ok(match *values { - [one_value] => one_value, - [v1, v2] if is_f32(v1) && is_f32(v2) => pack_f32s(v1, v2), - [v1, v2] if is_32(v1) && is_32(v2) => { - let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); - let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); - pack_i32s(v1, v2) - } - [v1, v2] => { - assert!(!(is_32(v1) && is_32(v2))); - build_struct( - func_type.get_return_type().unwrap().into_struct_type(), - &[v1, v2], - ) - } - [v1, v2, v3] if is_f32(v1) && is_f32(v2) => build_struct( - func_type.get_return_type().unwrap().into_struct_type(), - &[pack_f32s(v1, v2), v3], - ), - [v1, v2, v3] if is_32(v1) && is_32(v2) => { - let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); - let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); - build_struct( - func_type.get_return_type().unwrap().into_struct_type(), - &[pack_i32s(v1, v2), v3], - ) - } - [v1, v2, v3] if is_64(v1) && is_f32(v2) && is_f32(v3) => build_struct( - func_type.get_return_type().unwrap().into_struct_type(), - &[v1, pack_f32s(v2, v3)], - ), - [v1, v2, v3] if is_64(v1) && is_32(v2) && is_32(v3) => { - let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); - let v3 = builder.build_bitcast(v3, intrinsics.i32_ty, ""); - build_struct( - func_type.get_return_type().unwrap().into_struct_type(), - &[v1, pack_i32s(v2, v3)], - ) - } - [v1, v2, v3, v4] if is_32(v1) && is_32(v2) && is_32(v3) && is_32(v4) => { - let v1v2_pack = if is_f32(v1) && is_f32(v2) { - pack_f32s(v1, v2) - } else { - let v1 = builder.build_bitcast(v1, intrinsics.i32_ty, ""); - let v2 = builder.build_bitcast(v2, intrinsics.i32_ty, ""); - pack_i32s(v1, v2) - }; - let v3v4_pack = if is_f32(v3) && is_f32(v4) { - pack_f32s(v3, v4) - } else { - let v3 = builder.build_bitcast(v3, intrinsics.i32_ty, ""); - let v4 = builder.build_bitcast(v4, intrinsics.i32_ty, ""); - pack_i32s(v3, v4) - }; - build_struct( - func_type.get_return_type().unwrap().into_struct_type(), - &[v1v2_pack, v3v4_pack], - ) - } - _ => unreachable!("called to perform register return on struct return or void function"), - }) -} diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index c5e45334a7d..6195b023fb1 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -1,5 +1,4 @@ use super::{ - abi, intrinsics::{ tbaa_label, type_to_llvm, CtxType, FunctionCache, GlobalCache, Intrinsics, MemoryCache, }, @@ -22,6 +21,7 @@ use inkwell::{ }; use smallvec::SmallVec; +use crate::abi::{get_abi, Abi}; use crate::config::{CompiledKind, LLVM}; use crate::object_file::{load_object_file, CompiledFunction}; use wasmer_compiler::wasmparser::{MemoryImmediate, Operator}; @@ -57,13 +57,16 @@ fn const_zero(ty: BasicTypeEnum) -> BasicValueEnum { pub struct FuncTranslator { ctx: Context, target_machine: TargetMachine, + abi: Box, } impl FuncTranslator { pub fn new(target_machine: TargetMachine) -> Self { + let abi = get_abi(&target_machine); Self { ctx: Context::create(), target_machine, + abi, } } @@ -99,7 +102,9 @@ impl FuncTranslator { .unwrap(); let intrinsics = Intrinsics::declare(&module, &self.ctx); - let (func_type, func_attrs) = abi::func_type_to_llvm(&self.ctx, &intrinsics, wasm_fn_type)?; + let (func_type, func_attrs) = + self.abi + .func_type_to_llvm(&self.ctx, &intrinsics, wasm_fn_type)?; let func = module.add_function(&function_name, func_type, Some(Linkage::External)); for (attr, attr_loc) in &func_attrs { @@ -203,7 +208,7 @@ impl FuncTranslator { state, function: func, locals: params_locals, - ctx: CtxType::new(wasm_module, &func, &cache_builder), + ctx: CtxType::new(wasm_module, &func, &cache_builder, &*self.abi), unreachable_depth: 0, memory_styles, _table_styles, @@ -211,6 +216,7 @@ impl FuncTranslator { module_translation, wasm_module, symbol_registry, + abi: &*self.abi, }; fcg.ctx.add_func( func_index, @@ -1171,7 +1177,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { .map(|(v, i)| self.apply_pending_canonicalization(v, i)); if wasm_fn_type.results().is_empty() { self.builder.build_return(None); - } else if abi::is_sret(wasm_fn_type)? { + } else if self.abi.is_sret(wasm_fn_type)? { let sret = self .function .get_first_param() @@ -1198,7 +1204,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { self.builder.build_return(None); } else { self.builder - .build_return(Some(&abi::pack_values_for_register_return( + .build_return(Some(&self.abi.pack_values_for_register_return( &self.intrinsics, &self.builder, &results.collect::>(), @@ -1318,6 +1324,7 @@ pub struct LLVMFunctionCodeGenerator<'ctx, 'a> { module_translation: &'a ModuleTranslationState, wasm_module: &'a ModuleInfo, symbol_registry: &'a dyn SymbolRegistry, + abi: &'a dyn Abi, } impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { @@ -2138,7 +2145,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { _ => *v, }); - let params = abi::args_to_call( + let params = self.abi.args_to_call( &self.alloca_builder, func_type, callee_vmctx.into_pointer_value(), @@ -2186,7 +2193,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } */ - abi::rets_from_call(&self.builder, &self.intrinsics, call_site, func_type) + self.abi + .rets_from_call(&self.builder, &self.intrinsics, call_site, func_type) .iter() .for_each(|ret| self.state.push1(*ret)); } @@ -2357,7 +2365,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { self.builder.position_at_end(continue_block); let (llvm_func_type, llvm_func_attrs) = - abi::func_type_to_llvm(&self.context, &self.intrinsics, func_type)?; + self.abi + .func_type_to_llvm(&self.context, &self.intrinsics, func_type)?; let params = self.state.popn_save_extra(func_type.params().len())?; @@ -2381,7 +2390,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { _ => *v, }); - let params = abi::args_to_call( + let params = self.abi.args_to_call( &self.alloca_builder, func_type, ctx_ptr.into_pointer_value(), @@ -2436,7 +2445,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } */ - abi::rets_from_call(&self.builder, &self.intrinsics, call_site, func_type) + self.abi + .rets_from_call(&self.builder, &self.intrinsics, call_site, func_type) .iter() .for_each(|ret| self.state.push1(*ret)); } diff --git a/lib/compiler-llvm/src/translator/intrinsics.rs b/lib/compiler-llvm/src/translator/intrinsics.rs index 6d2de00cd7f..67ec724a413 100644 --- a/lib/compiler-llvm/src/translator/intrinsics.rs +++ b/lib/compiler-llvm/src/translator/intrinsics.rs @@ -4,7 +4,7 @@ //! //! [llvm-intrinsics]: https://llvm.org/docs/LangRef.html#intrinsic-functions -use super::abi; +use crate::abi::Abi; use inkwell::{ attributes::{Attribute, AttributeLoc}, builder::Builder, @@ -536,6 +536,7 @@ pub struct CtxType<'ctx, 'a> { wasm_module: &'a WasmerCompilerModule, cache_builder: &'a Builder<'ctx>, + abi: &'a dyn Abi, cached_memories: HashMap>, cached_tables: HashMap>, @@ -553,12 +554,14 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { wasm_module: &'a WasmerCompilerModule, func_value: &FunctionValue<'ctx>, cache_builder: &'a Builder<'ctx>, + abi: &'a dyn Abi, ) -> CtxType<'ctx, 'a> { CtxType { - ctx_ptr_value: abi::get_vmctx_ptr_param(func_value), + ctx_ptr_value: abi.get_vmctx_ptr_param(func_value), wasm_module, cache_builder, + abi, cached_memories: HashMap::new(), cached_tables: HashMap::new(), @@ -937,7 +940,7 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { Entry::Vacant(entry) => { debug_assert!(module.get_function(function_name).is_none()); let (llvm_func_type, llvm_func_attrs) = - abi::func_type_to_llvm(context, intrinsics, func_type)?; + self.abi.func_type_to_llvm(context, intrinsics, func_type)?; let func = module.add_function(function_name, llvm_func_type, Some(Linkage::External)); for (attr, attr_loc) in &llvm_func_attrs { @@ -970,7 +973,7 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { Entry::Occupied(entry) => entry.into_mut(), Entry::Vacant(entry) => { let (llvm_func_type, llvm_func_attrs) = - abi::func_type_to_llvm(context, intrinsics, func_type)?; + self.abi.func_type_to_llvm(context, intrinsics, func_type)?; debug_assert!(wasm_module.local_func_index(function_index).is_none()); let offset = offsets.vmctx_vmfunction_import(function_index); let offset = intrinsics.i32_ty.const_int(offset.into(), false); diff --git a/lib/compiler-llvm/src/translator/mod.rs b/lib/compiler-llvm/src/translator/mod.rs index bcf9e33e40c..ac546687d37 100644 --- a/lib/compiler-llvm/src/translator/mod.rs +++ b/lib/compiler-llvm/src/translator/mod.rs @@ -1,4 +1,3 @@ -pub mod abi; mod code; pub mod intrinsics; //mod stackmap; diff --git a/lib/compiler-llvm/src/translator/state.rs b/lib/compiler-llvm/src/translator/state.rs index ae0c35bbd9b..b1562183e52 100644 --- a/lib/compiler-llvm/src/translator/state.rs +++ b/lib/compiler-llvm/src/translator/state.rs @@ -74,10 +74,7 @@ impl<'ctx> ControlFrame<'ctx> { } pub fn is_loop(&self) -> bool { - match self { - ControlFrame::Loop { .. } => true, - _ => false, - } + matches!(self, ControlFrame::Loop { .. }) } }