From 773a9aa87063884ba8e212866736e48199d638d2 Mon Sep 17 00:00:00 2001 From: Zentrik Date: Wed, 10 Jul 2024 19:17:43 +0100 Subject: [PATCH] Revert "Remove references to non opaque pointers in codegen and LLVM passes (#54853)" This reverts commit 5e1bcdf9ff42ca17a995ce7bc7ef7fc30ce81d78. --- src/ccall.cpp | 64 ++++++------ src/cgutils.cpp | 160 +++++++++++++++++++----------- src/codegen.cpp | 175 ++++++++++++++++++++++----------- src/intrinsics.cpp | 37 ++++--- src/jitlayers.cpp | 3 +- src/llvm-alloc-opt.cpp | 27 ++--- src/llvm-codegen-shared.h | 34 ++++++- src/llvm-final-gc-lowering.cpp | 9 +- src/llvm-julia-licm.cpp | 3 +- src/llvm-late-gc-lowering.cpp | 16 +-- src/llvm-lower-handlers.cpp | 16 +-- src/llvm-pass-helpers.cpp | 9 +- src/llvm-ptls.cpp | 7 +- src/llvm-remove-addrspaces.cpp | 6 +- 14 files changed, 365 insertions(+), 201 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 03b539e70a378..12e0841844ce5 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -85,9 +85,9 @@ static bool runtime_sym_gvs(jl_codectx_t &ctx, const char *f_lib, const char *f_ runtime_lib = true; auto &libgv = ctx.emission_context.libMapGV[f_lib]; if (libgv.first == NULL) { - libptrgv = new GlobalVariable(*M, getPointerTy(M->getContext()), false, + libptrgv = new GlobalVariable(*M, getInt8PtrTy(M->getContext()), false, GlobalVariable::ExternalLinkage, - Constant::getNullValue(getPointerTy(M->getContext())), name); + Constant::getNullValue(getInt8PtrTy(M->getContext())), name); libgv.first = libptrgv; } else { @@ -169,7 +169,7 @@ static Value *runtime_sym_lookup( } else { // f_lib is actually one of the special sentinel values - libname = ConstantExpr::getIntToPtr(ConstantInt::get(emission_context.DL.getIntPtrType(irbuilder.getContext()), (uintptr_t)f_lib), getPointerTy(irbuilder.getContext())); + libname = ConstantExpr::getIntToPtr(ConstantInt::get(emission_context.DL.getIntPtrType(irbuilder.getContext()), (uintptr_t)f_lib), getInt8PtrTy(irbuilder.getContext())); } auto lookup = irbuilder.CreateCall(prepare_call_in(jl_builderModule(irbuilder), jldlsym_func), { libname, nameval, libptrgv }); @@ -186,7 +186,7 @@ static Value *runtime_sym_lookup( p->addIncoming(llvmf_orig, enter_bb); p->addIncoming(llvmf, llvmf->getParent()); setName(emission_context, p, f_name); - return p; + return irbuilder.CreateBitCast(p, funcptype); } static Value *runtime_sym_lookup( @@ -244,7 +244,7 @@ static GlobalVariable *emit_plt_thunk( auto T_pvoidfunc = JuliaType::get_pvoidfunc_ty(M->getContext()); GlobalVariable *got = new GlobalVariable(*M, T_pvoidfunc, false, GlobalVariable::ExternalLinkage, - plt, + ConstantExpr::getBitCast(plt, T_pvoidfunc), fname + "_got"); if (runtime_lib) { got->addAttribute("julia.libname", f_lib); @@ -256,7 +256,7 @@ static GlobalVariable *emit_plt_thunk( IRBuilder<> irbuilder(b0); Value *ptr = runtime_sym_lookup(ctx.emission_context, irbuilder, NULL, funcptype, f_lib, NULL, f_name, plt, libptrgv, llvmgv, runtime_lib); - StoreInst *store = irbuilder.CreateAlignedStore(ptr, got, Align(sizeof(void*))); + StoreInst *store = irbuilder.CreateAlignedStore(irbuilder.CreateBitCast(ptr, T_pvoidfunc), got, Align(sizeof(void*))); store->setAtomic(AtomicOrdering::Release); SmallVector args; for (auto &arg : plt->args()) @@ -311,6 +311,7 @@ static Value *emit_plt( GlobalVariable *libptrgv; GlobalVariable *llvmgv; bool runtime_lib = runtime_sym_gvs(ctx, f_lib, f_name, libptrgv, llvmgv); + PointerType *funcptype = PointerType::get(functype, 0); auto &pltMap = ctx.emission_context.allPltMap[attrs]; auto key = std::make_tuple(llvmgv, functype, cc); @@ -327,7 +328,7 @@ static Value *emit_plt( // since the only thing we do to this loaded pointer is to call it // immediately. got_val->setAtomic(AtomicOrdering::Unordered); - return got_val; + return ctx.builder.CreateBitCast(got_val, funcptype); } // --- ABI Implementations --- @@ -410,7 +411,7 @@ static Value *llvm_type_rewrite( assert(from_type->isPointerTy() == target_type->isPointerTy()); // expect that all ABIs consider all pointers to be equivalent if (target_type->isPointerTy()) - return v; + return emit_bitcast(ctx, v, target_type); // simple integer and float widening & conversion cases if (from_type->getPrimitiveSizeInBits() > 0 && @@ -446,13 +447,13 @@ static Value *llvm_type_rewrite( to = emit_static_alloca(ctx, target_type); setName(ctx.emission_context, to, "type_rewrite_buffer"); cast(to)->setAlignment(align); - from = to; + from = emit_bitcast(ctx, to, from_type->getPointerTo()); } else { from = emit_static_alloca(ctx, from_type); setName(ctx.emission_context, from, "type_rewrite_buffer"); cast(from)->setAlignment(align); - to = from; + to = emit_bitcast(ctx, from, target_type->getPointerTo()); } ctx.builder.CreateAlignedStore(v, from, align); auto pun = ctx.builder.CreateAlignedLoad(target_type, to, align); @@ -706,7 +707,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg interpret_symbol_arg(ctx, sym, args[1], /*ccall=*/false, false); if (sym.jl_ptr != NULL) { - res = sym.jl_ptr; + res = ctx.builder.CreateBitCast(sym.jl_ptr, lrt); } else if (sym.fptr != NULL) { res = ConstantInt::get(lrt, (uint64_t)sym.fptr); @@ -715,10 +716,10 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg } else if (sym.f_name != NULL) { if (sym.lib_expr) { - res = runtime_sym_lookup(ctx, getPointerTy(ctx.builder.getContext()), NULL, sym.lib_expr, sym.f_name, ctx.f); + res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), NULL, sym.lib_expr, sym.f_name, ctx.f); } else /*if (ctx.emission_context.imaging) */{ - res = runtime_sym_lookup(ctx, getPointerTy(ctx.builder.getContext()), sym.f_lib, NULL, sym.f_name, ctx.f); + res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); } } else { // Fall back to runtime intrinsic @@ -1231,7 +1232,7 @@ std::string generate_func_sig(const char *fname) bool isboxed; if (jl_is_abstract_ref_type(tti)) { tti = (jl_value_t*)jl_voidpointer_type; - t = getPointerTy(LLVMCtx); + t = getInt8PtrTy(LLVMCtx); isboxed = false; } else if (llvmcall && jl_is_llvmpointer_type(tti)) { @@ -1593,7 +1594,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) Value *retval; if (isboxed) { retval = boxed(ctx, argv[0]); - retval = emit_pointer_from_objref(ctx, retval /*T_prjlvalue*/); + retval = emit_pointer_from_objref(ctx, emit_bitcast(ctx, retval, ctx.types().T_prjlvalue)); } else { retval = emit_unbox(ctx, largty, argv[0], tti); @@ -1682,9 +1683,9 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) assert(lrt == getInt16Ty(ctx.builder.getContext())); assert(!isVa && !llvmcall && nccallargs == 0); JL_GC_POP(); - Value *ptask = get_current_task(ctx); + Value *ptask_i16 = emit_bitcast(ctx, get_current_task(ctx), getInt16PtrTy(ctx.builder.getContext())); const int tid_offset = offsetof(jl_task_t, tid); - Value *ptid = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptask, ConstantInt::get(ctx.types().T_size, tid_offset / sizeof(int8_t))); + Value *ptid = ctx.builder.CreateInBoundsGEP(getInt16Ty(ctx.builder.getContext()), ptask_i16, ConstantInt::get(ctx.types().T_size, tid_offset / sizeof(int16_t))); setName(ctx.emission_context, ptid, "thread_id_ptr"); LoadInst *tid = ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), ptid, Align(sizeof(int16_t))); setName(ctx.emission_context, tid, "thread_id"); @@ -1712,9 +1713,9 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) #endif ) { JL_GC_POP(); - Value *ptls_p = get_current_ptls(ctx); + Value *ptls_i32 = emit_bitcast(ctx, get_current_ptls(ctx), getInt32PtrTy(ctx.builder.getContext())); const int finh_offset = offsetof(jl_tls_states_t, finalizers_inhibited); - Value *pfinh = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptls_p, ConstantInt::get(ctx.types().T_size, finh_offset / sizeof(int8_t))); + Value *pfinh = ctx.builder.CreateInBoundsGEP(getInt32Ty(ctx.builder.getContext()), ptls_i32, ConstantInt::get(ctx.types().T_size, finh_offset / 4)); setName(ctx.emission_context, pfinh, "finalizers_inhibited_ptr"); LoadInst *finh = ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), pfinh, Align(sizeof(int32_t))); setName(ctx.emission_context, finh, "finalizers_inhibited"); @@ -1745,7 +1746,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) assert(lrt == getVoidTy(ctx.builder.getContext())); assert(!isVa && !llvmcall && nccallargs == 1); JL_GC_POP(); - Value *ptls_pv = get_current_ptls(ctx); + Value *ptls_pv = emit_bitcast(ctx, get_current_ptls(ctx), ctx.types().T_ppjlvalue); const int nt_offset = offsetof(jl_tls_states_t, next_task); Value *pnt = ctx.builder.CreateInBoundsGEP(ctx.types().T_pjlvalue, ptls_pv, ConstantInt::get(ctx.types().T_size, nt_offset / sizeof(void*))); setName(ctx.emission_context, pnt, "next_task_ptr"); @@ -1811,7 +1812,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) ++CCALL_STAT(jl_string_ptr); assert(lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); - auto obj = emit_pointer_from_objref(ctx, boxed(ctx, argv[0])); // T_pprjlvalue + auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])), + ctx.types().T_pprjlvalue); // The inbounds gep makes it more clear to LLVM that the resulting value is not // a null pointer. auto strp = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, obj, 1); @@ -1823,7 +1825,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) ++CCALL_STAT(jl_symbol_name); assert(lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); - auto obj = emit_pointer_from_objref(ctx, boxed(ctx, argv[0])); // T_pprjlvalue + auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])), + ctx.types().T_pprjlvalue); // The inbounds gep makes it more clear to LLVM that the resulting value is not // a null pointer. auto strp = ctx.builder.CreateConstInBoundsGEP1_32( @@ -1928,7 +1931,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) if (val.typ == (jl_value_t*)jl_symbol_type) { JL_GC_POP(); const int hash_offset = offsetof(jl_sym_t, hash); - Value *ph1 = decay_derived(ctx, boxed(ctx, val)); + Value *ph1 = emit_bitcast(ctx, decay_derived(ctx, boxed(ctx, val)), ctx.types().T_size->getPointerTo()); Value *ph2 = ctx.builder.CreateInBoundsGEP(ctx.types().T_size, ph1, ConstantInt::get(ctx.types().T_size, hash_offset / ctx.types().sizeof_ptr)); setName(ctx.emission_context, ph2, "object_id_ptr"); LoadInst *hashval = ctx.builder.CreateAlignedLoad(ctx.types().T_size, ph2, ctx.types().alignof_ptr); @@ -1940,13 +1943,15 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) else if (!val.isboxed) { // If the value is not boxed, try to compute the object id without // reboxing it. - auto T_p_derived = PointerType::get(ctx.builder.getContext(), AddressSpace::Derived); + auto T_pint8_derived = PointerType::get(getInt8Ty(ctx.builder.getContext()), AddressSpace::Derived); if (!val.isghost && !val.ispointer()) val = value_to_pointer(ctx, val); Value *args[] = { emit_typeof(ctx, val, false, true), - val.isghost ? ConstantPointerNull::get(T_p_derived) : - decay_derived(ctx, data_pointer(ctx, val)) + val.isghost ? ConstantPointerNull::get(T_pint8_derived) : + ctx.builder.CreateBitCast( + decay_derived(ctx, data_pointer(ctx, val)), + T_pint8_derived) }; Value *ret = ctx.builder.CreateCall(prepare_call(jl_object_id__func), ArrayRef(args)); setName(ctx.emission_context, ret, "object_id"); @@ -2040,7 +2045,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( result = emit_static_alloca(ctx, lrt); setName(ctx.emission_context, result, "ccall_sret"); sretty = lrt; - argvals[0] = result; + argvals[0] = ctx.builder.CreateBitCast(result, fargt_sig[0]); } else { // XXX: result needs to be zero'd and given a GC root here @@ -2051,7 +2056,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( sretty = ctx.types().T_jlvalue; sretboxed = true; gc_uses.push_back(result); - argvals[0] = emit_pointer_from_objref(ctx, result); + argvals[0] = ctx.builder.CreateBitCast(emit_pointer_from_objref(ctx, result), fargt_sig[0]); } } @@ -2114,7 +2119,8 @@ jl_cgval_t function_sig_t::emit_a_ccall( else if (symarg.jl_ptr != NULL) { ++LiteralCCalls; null_pointer_check(ctx, symarg.jl_ptr, nullptr); - llvmf = symarg.jl_ptr; + Type *funcptype = PointerType::get(functype, 0); + llvmf = ctx.builder.CreateBitCast(symarg.jl_ptr, funcptype); } else if (symarg.fptr != NULL) { ++LiteralCCalls; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 1d285cfaef937..de6023d50b800 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -307,6 +307,9 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) if (AS != AddressSpace::Tracked && AS != AddressSpace::Derived) return V; V = decay_derived(ctx, V); + Type *T = PointerType::get(ctx.types().T_jlvalue, AddressSpace::Derived); + if (V->getType() != T) + V = ctx.builder.CreateBitCast(V, T); Function *F = prepare_call(pointer_from_objref_func); CallInst *Call = ctx.builder.CreateCall(F, V); Call->setAttributes(F->getAttributes()); @@ -588,25 +591,31 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_binding_t *p) // bitcast a value, but preserve its address space when dealing with pointer types static Value *emit_bitcast(jl_codectx_t &ctx, Value *v, Type *jl_value) { - if (isa(jl_value)) { - return v; + if (isa(jl_value) && + v->getType()->getPointerAddressSpace() != jl_value->getPointerAddressSpace()) { + // Cast to the proper address space + #if JL_LLVM_VERSION >= 170000 + Type *jl_value_addr = PointerType::get(jl_value, v->getType()->getPointerAddressSpace()); + #else + Type *jl_value_addr = PointerType::getWithSamePointeeType(cast(jl_value), v->getType()->getPointerAddressSpace()); + #endif + ++EmittedPointerBitcast; + return ctx.builder.CreateBitCast(v, jl_value_addr); } else { return ctx.builder.CreateBitCast(v, jl_value); } } -// static Value *maybe_bitcast(jl_codectx_t &ctx, Value *V, Type *to) { -// if (isa(to)) { -// return V; -// } -// if (to != V->getType()) -// return emit_bitcast(ctx, V, to); -// return V; -// } +static Value *maybe_bitcast(jl_codectx_t &ctx, Value *V, Type *to) { + if (to != V->getType()) + return emit_bitcast(ctx, V, to); + return V; +} static Value *julia_binding_pvalue(jl_codectx_t &ctx, Value *bv) { + bv = emit_bitcast(ctx, bv, ctx.types().T_pprjlvalue); Value *offset = ConstantInt::get(ctx.types().T_size, offsetof(jl_binding_t, value) / ctx.types().sizeof_ptr); return ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, bv, offset); } @@ -694,7 +703,7 @@ static Type *bitstype_to_llvm(jl_value_t *bt, LLVMContext &ctxt, bool llvmcall = if (bt == (jl_value_t*)jl_bfloat16_type) return getBFloatTy(ctxt); if (jl_is_cpointer_type(bt)) - return PointerType::get(ctxt, 0); + return PointerType::get(getInt8Ty(ctxt), 0); if (jl_is_llvmpointer_type(bt)) { jl_value_t *as_param = jl_tparam1(bt); int as; @@ -704,7 +713,7 @@ static Type *bitstype_to_llvm(jl_value_t *bt, LLVMContext &ctxt, bool llvmcall = as = jl_unbox_int64(as_param); else jl_error("invalid pointer address space"); - return PointerType::get(ctxt, as); + return PointerType::get(getInt8Ty(ctxt), as); } int nb = jl_datatype_size(bt); return Type::getIntNTy(ctxt, nb * 8); @@ -1214,7 +1223,7 @@ static Value *emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull static Value *emit_datatype_types(jl_codectx_t &ctx, Value *dt) { - Value *Ptr = decay_derived(ctx, dt); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), ctx.types().T_ppjlvalue); Value *Idx = ConstantInt::get(ctx.types().T_size, offsetof(jl_datatype_t, types) / sizeof(void*)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto types = ai.decorateInst(ctx.builder.CreateAlignedLoad( @@ -1225,7 +1234,7 @@ static Value *emit_datatype_types(jl_codectx_t &ctx, Value *dt) static Value *emit_datatype_nfields(jl_codectx_t &ctx, Value *dt) { - Value *type_svec = emit_datatype_types(ctx, dt); + Value *type_svec = emit_bitcast(ctx, emit_datatype_types(ctx, dt), ctx.types().T_size->getPointerTo()); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto nfields = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_size, type_svec, Align(sizeof(void*)))); setName(ctx.emission_context, nfields, "datatype_nfields"); @@ -1236,17 +1245,17 @@ static Value *emit_datatype_nfields(jl_codectx_t &ctx, Value *dt) static Value *emit_datatype_size(jl_codectx_t &ctx, Value *dt, bool add_isunion=false) { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - Value *Ptr = decay_derived(ctx, dt); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), getInt32PtrTy(ctx.builder.getContext())->getPointerTo()); Value *Idx = ConstantInt::get(ctx.types().T_size, offsetof(jl_datatype_t, layout) / sizeof(int32_t*)); - Ptr = ctx.builder.CreateInBoundsGEP(getPointerTy(ctx.builder.getContext()), Ptr, Idx); - Ptr = ai.decorateInst(ctx.builder.CreateAlignedLoad(getPointerTy(ctx.builder.getContext()), Ptr, Align(sizeof(int32_t*)))); + Ptr = ctx.builder.CreateInBoundsGEP(getInt32PtrTy(ctx.builder.getContext()), Ptr, Idx); + Ptr = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt32PtrTy(ctx.builder.getContext()), Ptr, Align(sizeof(int32_t*)))); Idx = ConstantInt::get(ctx.types().T_size, offsetof(jl_datatype_layout_t, size) / sizeof(int32_t)); Value *SizePtr = ctx.builder.CreateInBoundsGEP(getInt32Ty(ctx.builder.getContext()), Ptr, Idx); Value *Size = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), SizePtr, Align(sizeof(int32_t)))); setName(ctx.emission_context, Size, "datatype_size"); if (add_isunion) { - Idx = ConstantInt::get(ctx.types().T_size, offsetof(jl_datatype_layout_t, flags) / sizeof(int8_t)); - Value *FlagPtr = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), Ptr, Idx); + Idx = ConstantInt::get(ctx.types().T_size, offsetof(jl_datatype_layout_t, flags) / sizeof(int16_t)); + Value *FlagPtr = ctx.builder.CreateInBoundsGEP(getInt16Ty(ctx.builder.getContext()), emit_bitcast(ctx, Ptr, getInt16PtrTy(ctx.builder.getContext())), Idx); Value *Flag = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), FlagPtr, Align(sizeof(int16_t)))); Flag = ctx.builder.CreateLShr(Flag, 4); Flag = ctx.builder.CreateAnd(Flag, ConstantInt::get(Flag->getType(), 1)); @@ -1308,10 +1317,10 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - Value *Ptr = decay_derived(ctx, dt); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), ctx.types().T_ppint8); Value *Idx = ConstantInt::get(ctx.types().T_size, offsetof(jl_datatype_t, name)); Value *Nam = ai.decorateInst( - ctx.builder.CreateAlignedLoad(getPointerTy(ctx.builder.getContext()), ctx.builder.CreateInBoundsGEP(getPointerTy(ctx.builder.getContext()), Ptr, Idx), Align(sizeof(int8_t*)))); + ctx.builder.CreateAlignedLoad(getInt8PtrTy(ctx.builder.getContext()), ctx.builder.CreateInBoundsGEP(getInt8PtrTy(ctx.builder.getContext()), Ptr, Idx), Align(sizeof(int8_t*)))); Value *Idx2 = ConstantInt::get(ctx.types().T_size, offsetof(jl_typename_t, n_uninitialized) + sizeof(((jl_typename_t*)nullptr)->n_uninitialized)); Value *mutabl = ai.decorateInst( ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), Nam, Idx2), Align(1))); @@ -1323,7 +1332,7 @@ static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) static Value *emit_datatype_isprimitivetype(jl_codectx_t &ctx, Value *typ) { Value *isprimitive; - isprimitive = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), decay_derived(ctx, typ), offsetof(jl_datatype_t, hash) + sizeof(((jl_datatype_t*)nullptr)->hash)); + isprimitive = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, decay_derived(ctx, typ), getInt8PtrTy(ctx.builder.getContext())), offsetof(jl_datatype_t, hash) + sizeof(((jl_datatype_t*)nullptr)->hash)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); isprimitive = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), isprimitive, Align(1))); isprimitive = ctx.builder.CreateLShr(isprimitive, 7); @@ -1337,7 +1346,7 @@ static Value *emit_datatype_name(jl_codectx_t &ctx, Value *dt) unsigned n = offsetof(jl_datatype_t, name) / sizeof(char*); Value *vptr = ctx.builder.CreateInBoundsGEP( ctx.types().T_pjlvalue, - maybe_decay_tracked(ctx, dt), + emit_bitcast(ctx, maybe_decay_tracked(ctx, dt), ctx.types().T_ppjlvalue), ConstantInt::get(ctx.types().T_size, n)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto name = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, vptr, Align(sizeof(void*)))); @@ -1538,6 +1547,7 @@ static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull, bool just // and we need to use it as an index to get the real object now Module *M = jl_Module; Value *smallp = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), prepare_global_in(M, jl_small_typeof_var), tag); + smallp = ctx.builder.CreateBitCast(smallp, typetag->getType()->getPointerTo(0)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); auto small = ctx.builder.CreateAlignedLoad(typetag->getType(), smallp, M->getDataLayout().getPointerABIAlignment(0)); small->setMetadata(LLVMContext::MD_nonnull, MDNode::get(M->getContext(), None)); @@ -1815,7 +1825,7 @@ static void emit_typecheck(jl_codectx_t &ctx, const jl_cgval_t &x, jl_value_t *t static Value *emit_isconcrete(jl_codectx_t &ctx, Value *typ) { Value *isconcrete; - isconcrete = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), decay_derived(ctx, typ), offsetof(jl_datatype_t, hash) + sizeof(((jl_datatype_t*)nullptr)->hash)); + isconcrete = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, decay_derived(ctx, typ), getInt8PtrTy(ctx.builder.getContext())), offsetof(jl_datatype_t, hash) + sizeof(((jl_datatype_t*)nullptr)->hash)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); isconcrete = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), isconcrete, Align(1))); isconcrete = ctx.builder.CreateLShr(isconcrete, 1); @@ -1867,7 +1877,7 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v else { // unboxed jl_value_t* Value *a = ainfo.V; if (ainfo.isghost) { - a = Constant::getNullValue(getPointerTy(ctx.builder.getContext())); + a = Constant::getNullValue(getInt8PtrTy(ctx.builder.getContext())); } else if (!ainfo.ispointer()) { // CreateAlloca is OK here since we are on an error branch @@ -1877,7 +1887,7 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v a = tempSpace; } ctx.builder.CreateCall(prepare_call(jluboundserror_func), { - decay_derived(ctx, a), + emit_bitcast(ctx, decay_derived(ctx, a), getInt8PtrTy(ctx.builder.getContext())), literal_pointer_val(ctx, ty), i }); } @@ -1945,7 +1955,7 @@ static void emit_lockstate_value(jl_codectx_t &ctx, Value *strct, bool newstate) { ++EmittedLockstates; if (strct->getType()->getPointerAddressSpace() == AddressSpace::Loaded) { - Value *v = strct; + Value *v = emit_bitcast(ctx, strct, PointerType::get(ctx.types().T_jlvalue, AddressSpace::Loaded)); ctx.builder.CreateCall(prepare_call(newstate ? jllockfield_func : jlunlockfield_func), v); } else { @@ -1990,7 +2000,12 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j if (nb != nb2) elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb2); } - Value *data = ptr; + Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); + Value *data; + if (ptr->getType() != ptrty) + data = emit_bitcast(ctx, ptr, ptrty); + else + data = ptr; if (idx_0based) data = ctx.builder.CreateInBoundsGEP(elty, data, idx_0based); Value *instr = nullptr; @@ -2033,7 +2048,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j if (elty != realelty) instr = ctx.builder.CreateTrunc(instr, realelty); if (intcast) { - ctx.builder.CreateStore(instr, intcast); + ctx.builder.CreateStore(instr, ctx.builder.CreateBitCast(intcast, instr->getType()->getPointerTo())); instr = nullptr; } } @@ -2143,6 +2158,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, r = ctx.builder.CreateZExt(r, elty); } } + Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); + if (ptr->getType() != ptrty) + ptr = ctx.builder.CreateBitCast(ptr, ptrty); if (isboxed) alignment = sizeof(void*); else if (!alignment) @@ -2270,7 +2288,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); if (intcast) { - ctx.builder.CreateStore(realCompare, intcast); + ctx.builder.CreateStore(realCompare, ctx.builder.CreateBitCast(intcast, realCompare->getType()->getPointerTo())); if (maybe_null_if_boxed) realCompare = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); } @@ -2356,7 +2374,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) realinstr = ctx.builder.CreateTrunc(realinstr, realelty); if (intcast) { - ctx.builder.CreateStore(realinstr, intcast); + ctx.builder.CreateStore(realinstr, ctx.builder.CreateBitCast(intcast, realinstr->getType()->getPointerTo())); oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); if (maybe_null_if_boxed) realinstr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); @@ -2430,7 +2448,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) instr = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, instr, realelty)); if (intcast) { - ctx.builder.CreateStore(instr, intcast); + ctx.builder.CreateStore(instr, ctx.builder.CreateBitCast(intcast, instr->getType()->getPointerTo())); instr = nullptr; } if (maybe_null_if_boxed) { @@ -2608,7 +2626,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, } Value *fldptr = ctx.builder.CreateInBoundsGEP( ctx.types().T_prjlvalue, - data_pointer(ctx, strct), + emit_bitcast(ctx, data_pointer(ctx, strct), ctx.types().T_pprjlvalue), idx0()); setName(ctx.emission_context, fldptr, "getfield_ptr"); LoadInst *fld = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, fldptr, Align(sizeof(void*))); @@ -2631,8 +2649,8 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, if (!stt->name->mutabl && !(maybe_null && (jft == (jl_value_t*)jl_bool_type || ((jl_datatype_t*)jft)->layout->npointers))) { // just compute the pointer and let user load it when necessary - Type *fty = julia_type_to_llvm(ctx, jft); //TODO: move this to a int8 GEP - Value *addr = ctx.builder.CreateInBoundsGEP(fty, ptr, idx); + Type *fty = julia_type_to_llvm(ctx, jft); + Value *addr = ctx.builder.CreateInBoundsGEP(fty, emit_bitcast(ctx, ptr, PointerType::get(fty, 0)), idx); *ret = mark_julia_slot(addr, jft, NULL, strct.tbaa); return true; } @@ -2846,7 +2864,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st if (byte_offset > 0) { addr = ctx.builder.CreateInBoundsGEP( getInt8Ty(ctx.builder.getContext()), - staddr, + emit_bitcast(ctx, staddr, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, byte_offset)); } else { @@ -2854,6 +2872,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st } } else { + staddr = maybe_bitcast(ctx, staddr, lt->getPointerTo()); if (jl_is_vecelement_type((jl_value_t*)jt)) addr = staddr; // VecElement types are unwrapped in LLVM. else if (isa(lt)) @@ -2866,7 +2885,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st } if (jl_field_isptr(jt, idx)) { setNameWithField(ctx.emission_context, addr, get_objname, jt, idx, Twine("_ptr")); - LoadInst *Load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, addr, Align(sizeof(void*))); + LoadInst *Load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, maybe_bitcast(ctx, addr, ctx.types().T_pprjlvalue), Align(sizeof(void*))); setNameWithField(ctx.emission_context, Load, get_objname, jt, idx, Twine()); Load->setOrdering(order <= jl_memory_order_notatomic ? AtomicOrdering::Unordered : get_llvm_atomic_order(order)); maybe_mark_load_dereferenceable(Load, maybe_null, jl_field_type(jt, idx)); @@ -2885,7 +2904,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st Value *ptindex; if (isboxed) { ptindex = ctx.builder.CreateConstInBoundsGEP1_32( - getInt8Ty(ctx.builder.getContext()), staddr, byte_offset + fsz1); + getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, staddr, getInt8PtrTy(ctx.builder.getContext())), byte_offset + fsz1); } else { ptindex = emit_struct_gep(ctx, cast(lt), staddr, byte_offset + fsz1); @@ -2951,6 +2970,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st // emit remaining bytes up to tindex if (i < ptindex - st_idx) { Value *staddr = ctx.builder.CreateConstInBoundsGEP1_32(ET, lv, i); + staddr = ctx.builder.CreateBitCast(staddr, getInt8PtrTy(ctx.builder.getContext())); for (; i < ptindex - st_idx; i++) { Value *fldv = ctx.builder.CreateExtractValue(obj, ArrayRef(st_idx + i)); Value *fldp = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), staddr, i); @@ -3017,6 +3037,7 @@ static Value *emit_genericmemoryelsize(jl_codectx_t &ctx, Value *v, jl_value_t * return ConstantInt::get(ctx.types().T_size, sz); } else { + v = emit_bitcast(ctx, v, ctx.types().T_prjlvalue); Value *t = emit_typeof(ctx, v, false, false, true); Value *elsize = emit_datatype_size(ctx, t, add_isunion); return ctx.builder.CreateZExt(elsize, ctx.types().T_size); @@ -3046,7 +3067,7 @@ static intptr_t genericmemoryype_maxsize(jl_value_t *ty) // the maxsize is stric static Value *emit_genericmemorylen(jl_codectx_t &ctx, Value *addr, jl_value_t *typ) { - addr = decay_derived(ctx, addr); + addr = emit_bitcast(ctx, decay_derived(ctx, addr), ctx.types().T_jlgenericmemory->getPointerTo()), addr = ctx.builder.CreateStructGEP(ctx.types().T_jlgenericmemory, addr, 0); LoadInst *LI = ctx.builder.CreateAlignedLoad(ctx.types().T_jlgenericmemory->getElementType(0), addr, Align(sizeof(size_t))); jl_aliasinfo_t aliasinfo_mem = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_memorylen); @@ -3060,7 +3081,9 @@ static Value *emit_genericmemorylen(jl_codectx_t &ctx, Value *addr, jl_value_t * static Value *emit_genericmemoryptr(jl_codectx_t &ctx, Value *mem, const jl_datatype_layout_t *layout, unsigned AS) { ++EmittedArrayptr; - Value *addr = mem; + PointerType *PT = cast(mem->getType()); + assert(PT == ctx.types().T_prjlvalue); + Value *addr = emit_bitcast(ctx, mem, ctx.types().T_jlgenericmemory->getPointerTo(PT->getAddressSpace())); addr = decay_derived(ctx, addr); addr = ctx.builder.CreateStructGEP(ctx.types().T_jlgenericmemory, addr, 1); setName(ctx.emission_context, addr, ".data_ptr"); @@ -3075,12 +3098,14 @@ static Value *emit_genericmemoryptr(jl_codectx_t &ctx, Value *mem, const jl_data assert(AS == AddressSpace::Loaded); ptr = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, ptr }); } + if (!layout->flags.arrayelem_isboxed) + ptr = ctx.builder.CreateBitCast(ptr, PointerType::get(getInt8Ty(ctx.builder.getContext()), AS)); return ptr; } static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t) { - Value *m = decay_derived(ctx, t); + Value *m = emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_jlgenericmemory->getPointerTo(0)); Value *addr = ctx.builder.CreateStructGEP(ctx.types().T_jlgenericmemory, m, 1); Type *T_data = ctx.types().T_jlgenericmemory->getElementType(1); LoadInst *LI = ctx.builder.CreateAlignedLoad(T_data, addr, Align(sizeof(char*))); @@ -3088,11 +3113,11 @@ static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t) LI->setMetadata(LLVMContext::MD_nonnull, MDNode::get(ctx.builder.getContext(), None)); jl_aliasinfo_t aliasinfo_mem = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_memoryown); aliasinfo_mem.decorateInst(LI); - addr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, m, JL_SMALL_BYTE_ALIGNMENT / sizeof(void*)); + addr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, emit_bitcast(ctx, m, LI->getType()), JL_SMALL_BYTE_ALIGNMENT / sizeof(void*)); Value *foreign = ctx.builder.CreateICmpNE(addr, decay_derived(ctx, LI)); return emit_guarded_test(ctx, foreign, t, [&] { addr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_jlgenericmemory, m, 1); - LoadInst *owner = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, addr, Align(sizeof(void*))); + LoadInst *owner = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, emit_bitcast(ctx, addr, ctx.types().T_pprjlvalue), Align(sizeof(void*))); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); ai.decorateInst(owner); return ctx.builder.CreateSelect(ctx.builder.CreateIsNull(owner), t, owner); @@ -3108,7 +3133,8 @@ static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tb { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); // newv should already be tagged - ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, Align(alignment))); + ai.decorateInst(ctx.builder.CreateAlignedStore(v, emit_bitcast(ctx, newv, + PointerType::get(v->getType(), 0)), Align(alignment))); } static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, MDNode *tbaa) @@ -3550,6 +3576,11 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab Value *decayed = decay_derived(ctx, box); AllocaInst *originalAlloca = cast(vinfo.V); box->takeName(originalAlloca); + #if JL_LLVM_VERSION >= 170000 + decayed = maybe_bitcast(ctx, decayed, PointerType::get(originalAlloca->getType(), AddressSpace::Derived)); + #else + decayed = maybe_bitcast(ctx, decayed, PointerType::getWithSamePointeeType(originalAlloca->getType(), AddressSpace::Derived)); + #endif // Warning: Very illegal IR here temporarily originalAlloca->mutateType(decayed->getType()); recursively_adjust_ptr_type(originalAlloca, 0, AddressSpace::Derived); @@ -3607,6 +3638,8 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con if (skip) tindex = ctx.builder.CreateSelect(skip, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), tindex); Value *src_ptr = data_pointer(ctx, src); + src_ptr = src_ptr ? maybe_bitcast(ctx, src_ptr, getInt8PtrTy(ctx.builder.getContext())) : src_ptr; + dest = maybe_bitcast(ctx, dest, getInt8PtrTy(ctx.builder.getContext())); BasicBlock *defaultBB = BasicBlock::Create(ctx.builder.getContext(), "union_move_skip", ctx.f); SwitchInst *switchInst = ctx.builder.CreateSwitch(tindex, defaultBB); BasicBlock *postBB = BasicBlock::Create(ctx.builder.getContext(), "post_union_move", ctx.f); @@ -3712,6 +3745,7 @@ static Value *emit_allocobj(jl_codectx_t &ctx, jl_datatype_t *jt, bool fully_ini // allocation for unknown object from an untracked pointer static Value *emit_new_bits(jl_codectx_t &ctx, Value *jt, Value *pval) { + pval = ctx.builder.CreateBitCast(pval, getInt8PtrTy(ctx.builder.getContext())); Function *F = prepare_call(jl_newbits_func); auto call = ctx.builder.CreateCall(F, { jt, pval }); call->setAttributes(F->getAttributes()); @@ -3731,9 +3765,9 @@ static void emit_write_barrier(jl_codectx_t &ctx, Value *parent, ArrayRef decay_ptrs; - decay_ptrs.push_back(maybe_decay_untracked(ctx, parent)); + decay_ptrs.push_back(maybe_decay_untracked(ctx, emit_bitcast(ctx, parent, ctx.types().T_prjlvalue))); for (auto ptr : ptrs) { - decay_ptrs.push_back(maybe_decay_untracked(ctx, ptr)); + decay_ptrs.push_back(maybe_decay_untracked(ctx, emit_bitcast(ctx, ptr, ctx.types().T_prjlvalue))); } ctx.builder.CreateCall(prepare_call(jl_write_barrier_func), decay_ptrs); } @@ -3852,7 +3886,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, if (byte_offset > 0) { addr = ctx.builder.CreateInBoundsGEP( getInt8Ty(ctx.builder.getContext()), - addr, + emit_bitcast(ctx, addr, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, byte_offset)); setNameWithField(ctx.emission_context, addr, get_objname, sty, idx0, Twine("_ptr")); } @@ -3861,7 +3895,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, if (!isboxed && jl_is_uniontype(jfty)) { size_t fsz1 = jl_field_size(sty, idx0) - 1; Value *ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), - addr, + emit_bitcast(ctx, addr, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, fsz1)); setNameWithField(ctx.emission_context, ptindex, get_objname, sty, idx0, Twine(".tindex_ptr")); return union_store(ctx, addr, ptindex, rhs, cmp, jfty, tbaa, ctx.tbaa().tbaa_unionselbyte, @@ -4018,6 +4052,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg // emit remaining bytes up to tindex if (i < ptindex - llvm_idx) { Value *staddr = ctx.builder.CreateConstInBoundsGEP1_32(ET, lv, i); + staddr = ctx.builder.CreateBitCast(staddr, getInt8PtrTy(ctx.builder.getContext())); for (; i < ptindex - llvm_idx; i++) { Value *fldp = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), staddr, i); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); @@ -4103,7 +4138,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); ai.decorateInst(ctx.builder.CreateAlignedStore( ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), - ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), strct, + ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, strct, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, jl_field_offset(sty, i) + jl_field_size(sty, i) - 1)), Align(1))); } @@ -4145,7 +4180,8 @@ static void emit_signal_fence(jl_codectx_t &ctx) static Value *emit_defer_signal(jl_codectx_t &ctx) { ++EmittedDeferSignal; - Value *ptls = get_current_ptls(ctx); + Value *ptls = emit_bitcast(ctx, get_current_ptls(ctx), + PointerType::get(ctx.types().T_sigatomic, 0)); Constant *offset = ConstantInt::getSigned(getInt32Ty(ctx.builder.getContext()), offsetof(jl_tls_states_t, defer_signal) / sizeof(sig_atomic_t)); return ctx.builder.CreateInBoundsGEP(ctx.types().T_sigatomic, ptls, ArrayRef(offset), "jl_defer_signal"); @@ -4194,7 +4230,7 @@ static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const if (ref.ispointer()) { LLVMContext &C = ctx.builder.getContext(); Type *type = get_memoryref_type(C, ctx.types().T_size, layout, 0); - LoadInst *load = ctx.builder.CreateLoad(type, data_pointer(ctx, ref)); + LoadInst *load = ctx.builder.CreateLoad(type, emit_bitcast(ctx, data_pointer(ctx, ref), PointerType::get(type, 0))); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ref.tbaa); ai.decorateInst(load); return load; @@ -4262,20 +4298,29 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg ovflw = ctx.builder.CreateICmpUGE(ctx.builder.CreateAdd(offset, mlen), ctx.builder.CreateNUWAdd(mlen, mlen)); } #endif - //Is this change fine - boffset = ctx.builder.CreateMul(offset, elsz); - newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), data, boffset); + boffset = ctx.builder.CreateMul(offset, elsz); +#if 0 // TODO: if opaque-pointers? + newdata = emit_bitcast(ctx, data, getInt8PtrTy(ctx.builder.getContext())); + newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), newdata, boffset); +#else + Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(ref.typ)); + newdata = emit_bitcast(ctx, data, elty->getPointerTo(0)); + newdata = ctx.builder.CreateInBoundsGEP(elty, newdata, offset); (void)boffset; // LLVM is very bad at handling GEP with types different from the load +#endif + newdata = emit_bitcast(ctx, newdata, data->getType()); if (bc) { BasicBlock *failBB, *endBB; failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); Value *mptr = emit_genericmemoryptr(ctx, mem, layout, 0); + mptr = emit_bitcast(ctx, mptr, newdata->getType()); #if 0 - Value *mend = mptr; + Value *mend = emit_bitcast(ctx, mptr, getInt8PtrTy(ctx.builder.getContext())); Value *blen = ctx.builder.CreateMul(mlen, elsz, "", true, true); - mend = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), mptr, blen); + mend = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), mend, blen); + mend = emit_bitcast(ctx, mend, newdata->getType()); Value *inbound = ctx.builder.CreateAnd( ctx.builder.CreateICmpULE(mptr, newdata), ctx.builder.CreateICmpULT(newdata, mend)); @@ -4321,6 +4366,7 @@ static jl_cgval_t emit_memoryref_offset(jl_codectx_t &ctx, const jl_cgval_t &ref else { Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); Value *mptr = emit_genericmemoryptr(ctx, mem, layout, 0); + mptr = emit_bitcast(ctx, mptr, mem->getType()); // (data - mptr) / elsz offset = ctx.builder.CreateSub( ctx.builder.CreatePtrToInt(data, ctx.types().T_size), @@ -4354,9 +4400,11 @@ static Value *emit_memoryref_ptr(jl_codectx_t &ctx, const jl_cgval_t &ref, const GEPlist.push_back(GEP); data = GEP->getPointerOperand()->stripPointerCastsSameRepresentation(); } + data = ctx.builder.CreateBitCast(data, ctx.types().T_pprjlvalue); data = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, data }); if (!GEPlist.empty()) { for (auto &GEP : make_range(GEPlist.rbegin(), GEPlist.rend())) { + data = ctx.builder.CreateBitCast(data, PointerType::get(GEP->getSourceElementType(), AS)); Instruction *GEP2 = GEP->clone(); GEP2->mutateType(PointerType::get(GEP->getResultElementType(), AS)); GEP2->setOperand(GetElementPtrInst::getPointerOperandIndex(), data); diff --git a/src/codegen.cpp b/src/codegen.cpp index b886d201e118c..227bff8222792 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -141,8 +141,23 @@ auto getVoidTy(LLVMContext &ctxt) { auto getCharTy(LLVMContext &ctxt) { return getInt32Ty(ctxt); } -auto getPointerTy(LLVMContext &ctxt) { - return PointerType::get(ctxt, 0); +auto getInt8PtrTy(LLVMContext &ctxt) { + return Type::getInt8PtrTy(ctxt); +} +auto getInt16PtrTy(LLVMContext &ctxt) { + return Type::getInt16PtrTy(ctxt); +} +auto getInt32PtrTy(LLVMContext &ctxt) { + return Type::getInt32PtrTy(ctxt); +} +auto getInt64PtrTy(LLVMContext &ctxt) { + return Type::getInt64PtrTy(ctxt); +} +auto getFloatPtrTy(LLVMContext &ctxt) { + return Type::getFloatPtrTy(ctxt); +} +auto getDoublePtrTy(LLVMContext &ctxt) { + return Type::getDoublePtrTy(ctxt); } typedef Instruction TerminatorInst; @@ -287,6 +302,7 @@ struct jl_typecache_t { IntegerType *T_sigatomic; + Type *T_ppint8; unsigned sizeof_ptr; Align alignof_ptr; @@ -296,7 +312,7 @@ struct jl_typecache_t { T_ptr(nullptr), T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr), T_ppjlvalue(nullptr), T_pprjlvalue(nullptr), T_jlgenericmemory(nullptr), T_jlarray(nullptr), T_pjlarray(nullptr), - T_jlfunc(nullptr), T_jlfuncparams(nullptr), T_sigatomic(nullptr), + T_jlfunc(nullptr), T_jlfuncparams(nullptr), T_sigatomic(nullptr), T_ppint8(nullptr), initialized(false) {} void initialize(LLVMContext &context, const DataLayout &DL) { @@ -304,7 +320,8 @@ struct jl_typecache_t { return; } initialized = true; - T_ptr = getPointerTy(context); + T_ptr = getInt8PtrTy(context); + T_ppint8 = PointerType::get(getInt8PtrTy(context), 0); T_sigatomic = Type::getIntNTy(context, sizeof(sig_atomic_t) * 8); T_size = DL.getIntPtrType(context); sizeof_ptr = DL.getPointerSize(); @@ -722,22 +739,22 @@ static AttributeList get_attrs_box_zext(LLVMContext &C, unsigned nbytes) static const auto jlRTLD_DEFAULT_var = new JuliaVariable{ XSTR(jl_RTLD_DEFAULT_handle), true, - [](Type *T_size) -> Type * { return getPointerTy(T_size->getContext()); }, + [](Type *T_size) -> Type * { return getInt8PtrTy(T_size->getContext()); }, }; static const auto jlexe_var = new JuliaVariable{ XSTR(jl_exe_handle), true, - [](Type *T_size) -> Type * { return getPointerTy(T_size->getContext()); }, + [](Type *T_size) -> Type * { return getInt8PtrTy(T_size->getContext()); }, }; static const auto jldll_var = new JuliaVariable{ XSTR(jl_libjulia_handle), true, - [](Type *T_size) -> Type * { return getPointerTy(T_size->getContext()); }, + [](Type *T_size) -> Type * { return getInt8PtrTy(T_size->getContext()); }, }; static const auto jldlli_var = new JuliaVariable{ XSTR(jl_libjulia_internal_handle), true, - [](Type *T_size) -> Type * { return getPointerTy(T_size->getContext()); }, + [](Type *T_size) -> Type * { return getInt8PtrTy(T_size->getContext()); }, }; static const auto jl_small_typeof_var = new JuliaVariable{ XSTR(jl_small_typeof), @@ -808,19 +825,19 @@ static const auto jlthrow_func = new JuliaFunction<>{ static const auto jlerror_func = new JuliaFunction<>{ XSTR(jl_error), [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), - {getPointerTy(C)}, false); }, + {getInt8PtrTy(C)}, false); }, get_attrs_noreturn, }; static const auto jlatomicerror_func = new JuliaFunction<>{ XSTR(jl_atomic_error), [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), - {getPointerTy(C)}, false); }, + {getInt8PtrTy(C)}, false); }, get_attrs_noreturn, }; static const auto jltypeerror_func = new JuliaFunction<>{ XSTR(jl_type_error), [](LLVMContext &C) { return FunctionType::get(getVoidTy(C), - {getPointerTy(C), JuliaType::get_prjlvalue_ty(C), PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, + {getInt8PtrTy(C), JuliaType::get_prjlvalue_ty(C), PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::CalleeRooted)}, false); }, get_attrs_noreturn, }; static const auto jlundefvarerror_func = new JuliaFunction<>{ @@ -1061,7 +1078,7 @@ static const auto jlenter_func = new JuliaFunction<>{ [](LLVMContext &C) { auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); return FunctionType::get(getVoidTy(C), - {T_pjlvalue, getPointerTy(C)}, false); }, + {T_pjlvalue, getInt8PtrTy(C)}, false); }, nullptr, }; static const auto jl_current_exception_func = new JuliaFunction<>{ @@ -1169,7 +1186,7 @@ static const auto jl_newbits_func = new JuliaFunction<>{ [](LLVMContext &C) { auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); return FunctionType::get(T_prjlvalue, - {T_prjlvalue, getPointerTy(C)}, false); + {T_prjlvalue, getInt8PtrTy(C)}, false); }, [](LLVMContext &C) { return AttributeList::get(C, AttributeSet(), @@ -1269,9 +1286,9 @@ static const auto setjmp_func = new JuliaFunction{ [](LLVMContext &C, const Triple &T) { if (T.isOSWindows()) return FunctionType::get(getInt32Ty(C), - {getPointerTy(C)}, false); + {getInt8PtrTy(C)}, false); return FunctionType::get(getInt32Ty(C), - {getPointerTy(C), getInt32Ty(C)}, false); + {getInt8PtrTy(C), getInt32Ty(C)}, false); }, [](LLVMContext &C) { return AttributeList::get(C, Attributes(C, {Attribute::ReturnsTwice}), @@ -1281,7 +1298,7 @@ static const auto setjmp_func = new JuliaFunction{ static const auto memcmp_func = new JuliaFunction{ XSTR(memcmp), [](LLVMContext &C, Type *T_size) { return FunctionType::get(getInt32Ty(C), - {getPointerTy(C), getPointerTy(C), T_size}, false); }, + {getInt8PtrTy(C), getInt8PtrTy(C), T_size}, false); }, [](LLVMContext &C) { AttrBuilder FnAttrs(C); #if JL_LLVM_VERSION >= 160000 @@ -1300,13 +1317,13 @@ static const auto memcmp_func = new JuliaFunction{ static const auto jldlsym_func = new JuliaFunction<>{ XSTR(jl_load_and_lookup), [](LLVMContext &C) { return FunctionType::get(JuliaType::get_pvoidfunc_ty(C), - {getPointerTy(C), getPointerTy(C), PointerType::get(getPointerTy(C), 0)}, false); }, + {getInt8PtrTy(C), getInt8PtrTy(C), PointerType::get(getInt8PtrTy(C), 0)}, false); }, nullptr, }; static const auto jllazydlsym_func = new JuliaFunction<>{ XSTR(jl_lazy_load_and_lookup), [](LLVMContext &C) { return FunctionType::get(JuliaType::get_pvoidfunc_ty(C), - {JuliaType::get_prjlvalue_ty(C), getPointerTy(C)}, false); }, + {JuliaType::get_prjlvalue_ty(C), getInt8PtrTy(C)}, false); }, nullptr, }; static const auto jltypeassert_func = new JuliaFunction<>{ @@ -1375,9 +1392,9 @@ static const auto jlgetcfunctiontrampoline_func = new JuliaFunction<>{ { T_prjlvalue, // f (object) T_pjlvalue, // result - getPointerTy(C), // cache + getInt8PtrTy(C), // cache T_pjlvalue, // fill - FunctionType::get(getPointerTy(C), { getPointerTy(C), T_ppjlvalue }, false)->getPointerTo(), // trampoline + FunctionType::get(getInt8PtrTy(C), { getInt8PtrTy(C), T_ppjlvalue }, false)->getPointerTo(), // trampoline T_pjlvalue, // env T_pprjlvalue, // vals }, false); @@ -1488,7 +1505,7 @@ static const auto except_enter_func = new JuliaFunction<>{ "julia.except_enter", [](LLVMContext &C) { auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); - auto RT = StructType::get(getInt32Ty(C), getPointerTy(C)); + auto RT = StructType::get(getInt32Ty(C), getInt8PtrTy(C)); return FunctionType::get(RT, {T_pjlvalue}, false); }, [](LLVMContext &C) { return AttributeList::get(C, Attributes(C, {Attribute::ReturnsTwice}), @@ -2201,6 +2218,7 @@ static void undef_derived_strct(jl_codectx_t &ctx, Value *ptr, jl_datatype_t *st return; size_t i, np = sty->layout->npointers; auto T_prjlvalue = JuliaType::get_prjlvalue_ty(ctx.builder.getContext()); + ptr = ctx.builder.CreateBitCast(ptr, T_prjlvalue->getPointerTo(ptr->getType()->getPointerAddressSpace())); for (i = 0; i < np; i++) { Value *fld = ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue, ptr, jl_ptr_offset(sty, i)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -2215,7 +2233,7 @@ static Value *emit_inttoptr(jl_codectx_t &ctx, Value *v, Type *ty) if (auto I = dyn_cast(v)) { auto ptr = I->getOperand(0); if (ty->getPointerAddressSpace() == ptr->getType()->getPointerAddressSpace()) - return ptr; + return ctx.builder.CreateBitCast(ptr, ty); #if JL_LLVM_VERSION >= 170000 else #else @@ -2593,7 +2611,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & tbaa = oldv.tbaa; slotv = ctx.builder.CreateSelect(isboxv, decay_derived(ctx, boxv), - decay_derived(ctx, slotv)); + decay_derived(ctx, emit_bitcast(ctx, slotv, boxv->getType()))); } jl_cgval_t newv = jl_cgval_t(slotv, false, typ, new_tindex, tbaa); assert(boxv->getType() == ctx.types().T_prjlvalue); @@ -2827,7 +2845,7 @@ static void visitLine(jl_codectx_t &ctx, uint64_t *ptr, Value *addend, const cha { Value *pv = ConstantExpr::getIntToPtr( ConstantInt::get(ctx.types().T_size, (uintptr_t)ptr), - getPointerTy(ctx.builder.getContext())); + getInt64PtrTy(ctx.builder.getContext())); Value *v = ctx.builder.CreateLoad(getInt64Ty(ctx.builder.getContext()), pv, true, name); v = ctx.builder.CreateAdd(v, addend); ctx.builder.CreateStore(v, pv, true); // volatile, not atomic, so this might be an underestimate, @@ -3464,8 +3482,8 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a gc_uses.append(get_gc_roots_for(ctx, arg2)); OperandBundleDef OpBundle("jl_roots", gc_uses); auto answer = ctx.builder.CreateCall(prepare_call(memcmp_func), { - varg1, - varg2, + ctx.builder.CreateBitCast(varg1, getInt8PtrTy(ctx.builder.getContext())), + ctx.builder.CreateBitCast(varg2, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, sz) }, ArrayRef(&OpBundle, gc_uses.empty() ? 0 : 1)); @@ -3488,6 +3506,7 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a } else if (sz > 512 && jl_struct_try_layout(sty) && sty->layout->flags.isbitsegal) { Type *TInt8 = getInt8Ty(ctx.builder.getContext()); + Type *TpInt8 = getInt8PtrTy(ctx.builder.getContext()); Type *TInt1 = getInt1Ty(ctx.builder.getContext()); Value *varg1 = arg1.ispointer() ? data_pointer(ctx, arg1) : value_to_pointer(ctx, arg1).V; @@ -3495,6 +3514,8 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a value_to_pointer(ctx, arg2).V; varg1 = emit_pointer_from_objref(ctx, varg1); varg2 = emit_pointer_from_objref(ctx, varg2); + varg1 = emit_bitcast(ctx, varg1, TpInt8); + varg2 = emit_bitcast(ctx, varg2, TpInt8); // See above for why we want to do this SmallVector gc_uses; @@ -3639,8 +3660,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva // not TIndex && not boxed implies it is an unboxed value of a different type from this singleton // (which was probably caught above, but just to be safe, we repeat it here explicitly) return ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0); - Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : arg1.Vboxed; - Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : arg2.Vboxed; + Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : maybe_bitcast(ctx, arg1.Vboxed, ctx.types().T_pjlvalue); + Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : maybe_bitcast(ctx, arg2.Vboxed, ctx.types().T_pjlvalue); // rooting these values isn't needed since we won't load this pointer // and we know at least one of them is a unique Singleton // which is already enough to ensure pointer uniqueness for this test @@ -3660,8 +3681,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva if (typ == jl_bool_type) { // aka jl_pointer_egal // some optimizations for bool, since pointer comparison may be better if ((arg1.isboxed || arg1.constant) && (arg2.isboxed || arg2.constant)) { // aka have-fast-pointer - Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : arg1.Vboxed; - Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : arg2.Vboxed; + Value *varg1 = arg1.constant ? literal_pointer_val(ctx, arg1.constant) : maybe_bitcast(ctx, arg1.Vboxed, ctx.types().T_pjlvalue); + Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : maybe_bitcast(ctx, arg2.Vboxed, ctx.types().T_pjlvalue); return ctx.builder.CreateICmpEQ(decay_derived(ctx, varg1), decay_derived(ctx, varg2)); } } @@ -4009,16 +4030,19 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); Value *data = emit_genericmemoryptr(ctx, mem, layout, AddressSpace::Loaded); Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); + data = emit_bitcast(ctx, data, AT->getPointerTo()); // compute tindex from val Value *ptindex; if (elsz == 0) { ptindex = data; } else { + data = emit_bitcast(ctx, data, AT->getPointerTo()); // isbits union selector bytes are stored after mem->length ptindex = ctx.builder.CreateInBoundsGEP(AT, data, mlen); data = ctx.builder.CreateInBoundsGEP(AT, data, idx0); } + ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx0); *ret = union_store(ctx, data, ptindex, val, cmp, ety, ctx.tbaa().tbaa_arraybuf, ctx.tbaa().tbaa_arrayselbyte, @@ -4034,6 +4058,7 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, assert(ptr); lock = ptr; // ptr += sizeof(lock); + ptr = emit_bitcast(ctx, ptr, getInt8PtrTy(ctx.builder.getContext())); ptr = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), ptr, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); } Value *data_owner = NULL; // owner object against which the write barrier must check @@ -4295,10 +4320,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } else { Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); + data = emit_bitcast(ctx, data, AT->getPointerTo()); // isbits union selector bytes are stored after mem->length bytes ptindex = ctx.builder.CreateInBoundsGEP(AT, data, mlen); data = ctx.builder.CreateInBoundsGEP(AT, data, idx0); } + ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx0); size_t elsz_c = 0, al_c = 0; int union_max = jl_islayout_inline(ety, &elsz_c, &al_c); @@ -4312,6 +4339,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, assert(ptr); lock = ptr; // ptr += sizeof(lock); + ptr = emit_bitcast(ctx, ptr, getInt8PtrTy(ctx.builder.getContext())); ptr = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), ptr, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); emit_lockstate_value(ctx, lock, true); } @@ -4403,8 +4431,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (needlock) { // n.b. no actual lock acquire needed, as the check itself only needs to load a single pointer and check for null // elem += sizeof(lock); + elem = emit_bitcast(ctx, elem, getInt8PtrTy(ctx.builder.getContext())); elem = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), elem, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); } + elem = emit_bitcast(ctx, elem, ctx.types().T_pprjlvalue); if (!isboxed) elem = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, elem, layout->first_ptr); // emit this using the same type as jl_builtin_memoryrefget @@ -4692,7 +4722,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (nargs == 3) emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "fieldtype"); emit_bounds_check(ctx, typ, (jl_value_t*)jl_datatype_type, idx, types_len, boundscheck); - Value *fieldtyp_p = ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, decay_derived(ctx, types_svec), idx); + Value *fieldtyp_p = ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, decay_derived(ctx, emit_bitcast(ctx, types_svec, ctx.types().T_pprjlvalue)), idx); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *fieldtyp = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, fieldtyp_p, Align(sizeof(void*)))); setName(ctx.emission_context, fieldtyp, "fieldtype"); @@ -4719,7 +4749,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } // String and SimpleVector's length fields have the same layout - auto ptr = boxed(ctx, obj); + auto ptr = emit_bitcast(ctx, boxed(ctx, obj), ctx.types().T_size->getPointerTo()); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *len = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_size, ptr, ctx.types().alignof_ptr)); MDBuilder MDB(ctx.builder.getContext()); @@ -4843,7 +4873,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, auto tbaa = best_field_tbaa(ctx, obj, stt, fieldidx, offs); if (!jl_field_isptr(stt, fieldidx)) offs += ((jl_datatype_t*)jl_field_type(stt, fieldidx))->layout->first_ptr; - Value *ptr = data_pointer(ctx, obj); + Value *ptr = emit_bitcast(ctx, data_pointer(ctx, obj), ctx.types().T_pprjlvalue); Value *addr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, ptr, offs); // emit this using the same type as emit_getfield_knownidx // so that LLVM may be able to load-load forward them and fold the result @@ -4991,12 +5021,13 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos // n.b.: specTypes is required to be a datatype by construction for specsig jl_cgval_t arg = argv[i]; if (is_opaque_closure && i == 0) { + Type *at = cft->getParamType(idx); // Special optimization for opaque closures: We know that specsig opaque // closures don't look at their type tag (they are fairly quickly discarded // for their environments). Therefore, we can just pass these as a pointer, // rather than a boxed value. arg = value_to_pointer(ctx, arg); - argvals[idx] = decay_derived(ctx, data_pointer(ctx, arg)); + argvals[idx] = decay_derived(ctx, maybe_bitcast(ctx, data_pointer(ctx, arg), at)); } else if (is_uniquerep_Type(jt)) { continue; @@ -5006,15 +5037,19 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos if (type_is_ghost(et)) continue; assert(idx < nfargs); + Type *at = cft->getParamType(idx); if (isboxed) { + assert(at == ctx.types().T_prjlvalue && et == ctx.types().T_prjlvalue); argvals[idx] = boxed(ctx, arg); } else if (et->isAggregateType()) { arg = value_to_pointer(ctx, arg); // can lazy load on demand, no copy needed - argvals[idx] = decay_derived(ctx, data_pointer(ctx, arg)); + assert(at == PointerType::get(et, AddressSpace::Derived)); + argvals[idx] = decay_derived(ctx, maybe_bitcast(ctx, data_pointer(ctx, arg), at)); } else { + assert(at == et); Value *val = emit_unbox(ctx, et, arg, jt); if (!val) { // There was a type mismatch of some sort - exit early @@ -5067,7 +5102,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos ctx.builder.CreateICmpEQ( ctx.builder.CreateAnd(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), UNION_BOX_MARKER)), ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0)), - decay_derived(ctx, argvals[0]), + decay_derived(ctx, ctx.builder.CreateBitCast(argvals[0], ctx.types().T_pjlvalue)), decay_derived(ctx, box) ); retval = mark_julia_slot(derived, @@ -5378,7 +5413,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo if (f.typ == (jl_value_t*)jl_intrinsic_type) { fptr = prepare_call(jlintrinsic_func); F = f.ispointer() ? data_pointer(ctx, f) : value_to_pointer(ctx, f).V; - F = decay_derived(ctx, F); + F = decay_derived(ctx, maybe_bitcast(ctx, F, ctx.types().T_pjlvalue)); cc = julia_call3; } else { @@ -5685,7 +5720,8 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va if (vi.usedUndef) isnull = ctx.builder.CreateSelect(load_unbox, isnull, box_isnull); if (v.V) { // v.V will be null if it is a union of all ghost values - v.V = ctx.builder.CreateSelect(load_unbox, decay_derived(ctx, v.V), decay_derived(ctx, boxed)); + v.V = ctx.builder.CreateSelect(load_unbox, emit_bitcast(ctx, + decay_derived(ctx, v.V), boxed->getType()), decay_derived(ctx, boxed)); } else v.V = boxed; v.Vboxed = boxed; @@ -5743,6 +5779,9 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu if (vi.pTIndex) // TODO: use lifetime-end here instead ctx.builder.CreateStore(UndefValue::get(cast(vi.value.V)->getAllocatedType()), vi.value.V); Type *store_ty = julia_type_to_llvm(ctx, rval_info.constant ? jl_typeof(rval_info.constant) : rval_info.typ); + Type *dest_ty = store_ty->getPointerTo(); + if (dest_ty != dest->getType()) + dest = emit_bitcast(ctx, dest, dest_ty); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); ai.decorateInst(ctx.builder.CreateStore( emit_unbox(ctx, store_ty, rval_info, rval_info.typ), @@ -5812,8 +5851,8 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) ctx.builder.CreateMemCpy(phi, MaybeAlign(min_align), dest, dest->getAlign(), nbytes, false); ctx.builder.CreateLifetimeEnd(dest); Value *ptr = ctx.builder.CreateSelect(isboxed, - decay_derived(ctx, ptr_phi), - decay_derived(ctx, phi)); + maybe_bitcast(ctx, decay_derived(ctx, ptr_phi), getInt8PtrTy(ctx.builder.getContext())), + maybe_bitcast(ctx, decay_derived(ctx, phi), getInt8PtrTy(ctx.builder.getContext()))); jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, best_tbaa(ctx.tbaa(), phiType)); val.Vboxed = ptr_phi; ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, r)); @@ -6705,9 +6744,14 @@ static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0, bool or_new=fal ctx.pgcstack->setName("pgcstack"); } +static Value *get_current_task(jl_codectx_t &ctx, Type *T) +{ + return emit_bitcast(ctx, get_current_task_from_pgcstack(ctx.builder, ctx.types().T_size, ctx.pgcstack), T); +} + static Value *get_current_task(jl_codectx_t &ctx) { - return get_current_task_from_pgcstack(ctx.builder, ctx.types().T_size, ctx.pgcstack); + return get_current_task(ctx, ctx.types().T_pjlvalue); } // Get PTLS through current task. @@ -6719,7 +6763,7 @@ static Value *get_current_ptls(jl_codectx_t &ctx) // Get the address of the world age of the current task static Value *get_tls_world_age_field(jl_codectx_t &ctx) { - Value *ct = get_current_task(ctx); + Value *ct = get_current_task(ctx, ctx.types().T_size->getPointerTo()); return ctx.builder.CreateInBoundsGEP( ctx.types().T_size, ct, @@ -6729,7 +6773,7 @@ static Value *get_tls_world_age_field(jl_codectx_t &ctx) static Value *get_scope_field(jl_codectx_t &ctx) { - Value *ct = get_current_task(ctx); + Value *ct = get_current_task(ctx, ctx.types().T_prjlvalue->getPointerTo()); return ctx.builder.CreateInBoundsGEP( ctx.types().T_prjlvalue, ct, @@ -6862,6 +6906,7 @@ static void emit_cfunc_invalidate( ctx.builder.CreateRetVoid(); } else { + gf_ret = emit_bitcast(ctx, gf_ret, gfrt->getPointerTo()); ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gfrt, gf_ret, Align(julia_alignment(rettype)))); } break; @@ -7057,7 +7102,7 @@ static Function* gen_cfun_wrapper( ctx.types().T_size, ctx.builder.CreateConstInBoundsGEP1_32( ctx.types().T_size, - literal_pointer_val(ctx, (jl_value_t*)codeinst), + emit_bitcast(ctx, literal_pointer_val(ctx, (jl_value_t*)codeinst), ctx.types().T_size->getPointerTo()), offsetof(jl_code_instance_t, max_world) / ctx.types().sizeof_ptr), ctx.types().alignof_ptr); age_ok = ctx.builder.CreateICmpUGE(lam_max, world_v); @@ -7111,7 +7156,7 @@ static Function* gen_cfun_wrapper( if (aref) { if (jargty == (jl_value_t*)jl_any_type) { inputarg = mark_julia_type(ctx, - ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, val, Align(sizeof(void*))), + ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, emit_bitcast(ctx, val, ctx.types().T_pprjlvalue), Align(sizeof(void*))), true, jl_any_type); } else if (static_at && jl_is_concrete_immutable(jargty)) { // anything that could be stored unboxed @@ -7123,13 +7168,14 @@ static Function* gen_cfun_wrapper( inputarg = ghostValue(ctx, jargty); } else { + val = emit_bitcast(ctx, val, T->getPointerTo()); val = ctx.builder.CreateAlignedLoad(T, val, Align(1)); // make no alignment assumption about pointer from C inputarg = mark_julia_type(ctx, val, false, jargty); } } else if (static_at || (!jl_is_typevar(jargty) && !jl_is_immutable_datatype(jargty))) { // must be a jl_value_t* (because it's mutable or contains gc roots) - inputarg = mark_julia_type(ctx, maybe_decay_untracked(ctx, val), true, jargty_proper); + inputarg = mark_julia_type(ctx, maybe_decay_untracked(ctx, emit_bitcast(ctx, val, ctx.types().T_prjlvalue)), true, jargty_proper); } else { // allocate val into a new box, if it might not be boxed @@ -7149,15 +7195,16 @@ static Function* gen_cfun_wrapper( Value *isrtboxed = ctx.builder.CreateIsNull(val); // XXX: this is the wrong condition and should be inspecting runtime_dt instead ctx.builder.CreateCondBr(isrtboxed, boxedBB, loadBB); ctx.builder.SetInsertPoint(boxedBB); - Value *p1 = val; + Value *p1 = ctx.builder.CreateBitCast(val, ctx.types().T_pjlvalue); p1 = track_pjlvalue(ctx, p1); ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(loadBB); Value *isrtany = ctx.builder.CreateICmpEQ( - literal_pointer_val(ctx, (jl_value_t*)jl_any_type), val); + literal_pointer_val(ctx, (jl_value_t*)jl_any_type), + ctx.builder.CreateBitCast(val, ctx.types().T_pjlvalue)); ctx.builder.CreateCondBr(isrtany, isanyBB, unboxedBB); ctx.builder.SetInsertPoint(isanyBB); - Value *p2 = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, val, Align(sizeof(void*))); + Value *p2 = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateBitCast(val, ctx.types().T_pprjlvalue), Align(sizeof(void*))); ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(unboxedBB); Value *p3 = emit_new_bits(ctx, runtime_dt, val); @@ -7333,7 +7380,8 @@ static Function* gen_cfun_wrapper( else if (T->isAggregateType()) { // aggregate types are passed by pointer inputarg = value_to_pointer(ctx, inputarg); - arg = decay_derived(ctx, data_pointer(ctx, inputarg)); + arg = maybe_bitcast(ctx, decay_derived(ctx, data_pointer(ctx, inputarg)), + T->getPointerTo()); } else { arg = emit_unbox(ctx, T, inputarg, spect); @@ -7386,7 +7434,7 @@ static Function* gen_cfun_wrapper( ctx.builder.CreateICmpEQ( ctx.builder.CreateAnd(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), UNION_BOX_MARKER)), ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0)), - decay_derived(ctx, result), + decay_derived(ctx, ctx.builder.CreateBitCast(result, ctx.types().T_pjlvalue)), decay_derived(ctx, box)); retval = mark_julia_slot(derived, astrt, @@ -7445,7 +7493,7 @@ static Function* gen_cfun_wrapper( if (nest) { funcName += "make"; Function *cw_make = Function::Create( - FunctionType::get(getPointerTy(ctx.builder.getContext()), { getPointerTy(ctx.builder.getContext()), ctx.types().T_ppjlvalue }, false), + FunctionType::get(getInt8PtrTy(ctx.builder.getContext()), { getInt8PtrTy(ctx.builder.getContext()), ctx.types().T_ppjlvalue }, false), GlobalVariable::ExternalLinkage, funcName, M); jl_init_function(cw_make, ctx.emission_context.TargetTriple); @@ -7460,8 +7508,8 @@ static Function* gen_cfun_wrapper( Function *adjust_trampoline = Intrinsic::getDeclaration(cw_make->getParent(), Intrinsic::adjust_trampoline); cwbuilder.CreateCall(init_trampoline, { Tramp, - cw, - NVal, + cwbuilder.CreateBitCast(cw, getInt8PtrTy(ctx.builder.getContext())), + cwbuilder.CreateBitCast(NVal, getInt8PtrTy(ctx.builder.getContext())) }); cwbuilder.CreateRet(cwbuilder.CreateCall(adjust_trampoline, { Tramp })); cw = cw_make; @@ -7603,7 +7651,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con F = ctx.builder.CreateCall(prepare_call(jlgetcfunctiontrampoline_func), { fobj, literal_pointer_val(ctx, output_type), - cache, + ctx.builder.CreateBitCast(cache, getInt8PtrTy(ctx.builder.getContext())), literal_pointer_val(ctx, (jl_value_t*)fill), F, closure_types ? literal_pointer_val(ctx, (jl_value_t*)unionall_env) : Constant::getNullValue(ctx.types().T_pjlvalue), @@ -7618,7 +7666,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con assert(jl_datatype_size(output_type) == sizeof(void*) * 4); Value *strct = emit_allocobj(ctx, (jl_datatype_t*)output_type, true); setName(ctx.emission_context, strct, "cfun_result"); - Value *derived_strct = decay_derived(ctx, strct); + Value *derived_strct = emit_bitcast(ctx, decay_derived(ctx, strct), ctx.types().T_size->getPointerTo()); MDNode *tbaa = best_tbaa(ctx.tbaa(), output_type); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.decorateInst(ctx.builder.CreateStore(F, derived_strct)); @@ -7789,7 +7837,7 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret ty)); } if (!isboxed) { - theArg = decay_derived(ctx, theArg); + theArg = decay_derived(ctx, emit_bitcast(ctx, theArg, PointerType::get(lty, 0))); if (!lty->isAggregateType()) // keep "aggregate" type values in place as pointers theArg = ctx.builder.CreateAlignedLoad(lty, theArg, Align(julia_alignment(ty))); } @@ -8013,6 +8061,8 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value else { if (fval->getType()->isIntegerTy()) fval = emit_inttoptr(ctx, fval, ftype->getPointerTo()); + else + fval = emit_bitcast(ctx, fval, ftype->getPointerTo()); } if (auto F = dyn_cast(fval)) { if (gcstack_arg) @@ -8036,6 +8086,10 @@ static void emit_sret_roots(jl_codectx_t &ctx, bool isptr, Value *Src, Type *T, static DISubroutineType * get_specsig_di(jl_codectx_t &ctx, jl_debugcache_t &debuginfo, jl_value_t *rt, jl_value_t *sig, DIBuilder &dbuilder) { + #if JL_LLVM_VERSION < 170000 + if (isptr && !cast(Src->getType())->isOpaqueOrPointeeTypeMatches(T)) + Src = ctx.builder.CreateBitCast(Src, T->getPointerTo(Src->getType()->getPointerAddressSpace())); + #endif size_t nargs = jl_nparams(sig); // TODO: if this is a Varargs function, our debug info for the `...` var may be misleading SmallVector ditypes(nargs + 1); ditypes[0] = julia_type_to_di(ctx, debuginfo, rt, &dbuilder, false); @@ -8730,7 +8784,7 @@ static jl_llvm_functions_t if (i == 0 && ctx.is_opaque_closure) { // Load closure world Value *oc_this = decay_derived(ctx, &*AI++); - Value *argaddr = oc_this; + Value *argaddr = emit_bitcast(ctx, oc_this, getInt8PtrTy(ctx.builder.getContext())); Value *worldaddr = ctx.builder.CreateInBoundsGEP( getInt8Ty(ctx.builder.getContext()), argaddr, ConstantInt::get(ctx.types().T_size, offsetof(jl_opaque_closure_t, world))); @@ -9290,11 +9344,14 @@ static jl_llvm_functions_t } else { Type *store_ty = retvalinfo.V->getType(); + Type *dest_ty = store_ty->getPointerTo(); Value *Val = retvalinfo.V; if (returninfo.return_roots) { assert(julia_type_to_llvm(ctx, retvalinfo.typ) == store_ty); emit_sret_roots(ctx, false, Val, store_ty, f->arg_begin() + 1, get_returnroots_type(ctx, returninfo.return_roots), returninfo.return_roots); } + if (dest_ty != sret->getType()) + sret = emit_bitcast(ctx, sret, dest_ty); ctx.builder.CreateAlignedStore(Val, sret, Align(julia_alignment(retvalinfo.typ))); assert(retvalinfo.TIndex == NULL && "unreachable"); // unimplemented representation } @@ -9406,7 +9463,7 @@ static jl_llvm_functions_t ctx.builder.SetInsertPoint(tryblk); auto ehptr = ctx.builder.CreateInBoundsGEP( ctx.types().T_ptr, - ct, + emit_bitcast(ctx, ct, ctx.types().T_ppint8), ConstantInt::get(ctx.types().T_size, offsetof(jl_task_t, eh) / ctx.types().sizeof_ptr), "eh"); ctx.builder.CreateAlignedStore(ehbuf, ehptr, ctx.types().alignof_ptr); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index ad89abf6ca1a2..d6d5ff837d79e 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -403,23 +403,26 @@ static Value *emit_unboxed_coercion(jl_codectx_t &ctx, Type *to, Value *unboxed) CreateTrap(ctx.builder); return UndefValue::get(to); } + if (frompointer && topointer) { + unboxed = emit_bitcast(ctx, unboxed, to); + } else if (!ty->isIntOrPtrTy() && !ty->isFloatingPointTy()) { assert(DL.getTypeSizeInBits(ty) == DL.getTypeSizeInBits(to)); AllocaInst *cast = ctx.builder.CreateAlloca(ty); setName(ctx.emission_context, cast, "coercion"); ctx.builder.CreateStore(unboxed, cast); - unboxed = ctx.builder.CreateLoad(to, cast); + unboxed = ctx.builder.CreateLoad(to, ctx.builder.CreateBitCast(cast, to->getPointerTo())); } else if (frompointer) { Type *INTT_to = INTT(to, DL); unboxed = ctx.builder.CreatePtrToInt(unboxed, INTT_to); setName(ctx.emission_context, unboxed, "coercion"); - if (INTT_to != to) //TODO when is this true? + if (INTT_to != to) unboxed = ctx.builder.CreateBitCast(unboxed, to); } else if (topointer) { Type *INTT_to = INTT(to, DL); - if (to != INTT_to) //TODO when is this true? + if (to != INTT_to) unboxed = ctx.builder.CreateBitCast(unboxed, INTT_to); unboxed = emit_inttoptr(ctx, unboxed, to); setName(ctx.emission_context, unboxed, "coercion"); @@ -457,7 +460,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va if (jt == (jl_value_t*)jl_bool_type || to->isIntegerTy(1)) { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); - Instruction *unbox_load = ai.decorateInst(ctx.builder.CreateLoad(getInt8Ty(ctx.builder.getContext()), p)); + Instruction *unbox_load = ai.decorateInst(ctx.builder.CreateLoad(getInt8Ty(ctx.builder.getContext()), maybe_bitcast(ctx, p, getInt8PtrTy(ctx.builder.getContext())))); setName(ctx.emission_context, unbox_load, p->getName() + ".unbox"); if (jt == (jl_value_t*)jl_bool_type) unbox_load->setMetadata(LLVMContext::MD_range, MDNode::get(ctx.builder.getContext(), { @@ -491,6 +494,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va return emit_unboxed_coercion(ctx, to, ai.decorateInst(load)); } } + p = maybe_bitcast(ctx, p, ptype); Instruction *load = ctx.builder.CreateAlignedLoad(to, p, Align(alignment)); setName(ctx.emission_context, load, p->getName() + ".unbox"); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); @@ -510,6 +514,9 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest if (!x.ispointer()) { // already unboxed, but sometimes need conversion (e.g. f32 -> i32) assert(x.V); Value *unboxed = zext_struct(ctx, x.V); + Type *dest_ty = unboxed->getType()->getPointerTo(); + if (dest->getType() != dest_ty) + dest = emit_bitcast(ctx, dest, dest_ty); StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, Align(alignment)); store->setVolatile(isVolatile); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest); @@ -610,7 +617,8 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, v.tbaa); vx = ai.decorateInst(ctx.builder.CreateLoad( storage_type, - data_pointer(ctx, v))); + emit_bitcast(ctx, data_pointer(ctx, v), + storage_type->getPointerTo()))); setName(ctx.emission_context, vx, "bitcast"); } @@ -632,7 +640,11 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) setName(ctx.emission_context, vx, "bitcast_coercion"); } else if (vxt->isPointerTy() && llvmt->isPointerTy()) { // emit_bitcast preserves the origin address space, which we can't have here + #if JL_LLVM_VERSION >= 170000 vx = ctx.builder.CreateAddrSpaceCast(vx, llvmt); + #else + vx = ctx.builder.CreatePointerBitCastOrAddrSpaceCast(vx, llvmt); + #endif if (isa(vx) && !vx->hasName()) // cast may have been folded setName(ctx.emission_context, vx, "bitcast_coercion"); @@ -766,8 +778,8 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) im1 = ctx.builder.CreateMul(im1, ConstantInt::get(ctx.types().T_size, LLT_ALIGN(size, jl_datatype_align(ety)))); setName(ctx.emission_context, im1, "pointerref_offset"); - Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); - thePtr = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1); + Value *thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); + thePtr = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, thePtr, getInt8PtrTy(ctx.builder.getContext())), im1); setName(ctx.emission_context, thePtr, "pointerref_src"); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); emit_memcpy(ctx, strct, jl_aliasinfo_t::fromTBAA(ctx, tbaa), thePtr, jl_aliasinfo_t::fromTBAA(ctx, nullptr), size, sizeof(jl_value_t*), align_nb); @@ -840,7 +852,7 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) ai.decorateInst(store); } else if (x.ispointer()) { - thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); + thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); uint64_t size = jl_datatype_size(ety); im1 = ctx.builder.CreateMul(im1, ConstantInt::get(ctx.types().T_size, LLT_ALIGN(size, jl_datatype_align(ety)))); @@ -949,15 +961,16 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, ArrayRef assert(jl_is_datatype(ety)); Value *strct = emit_allocobj(ctx, (jl_datatype_t*)ety, true); setName(ctx.emission_context, strct, "atomic_pointerref_box"); - Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); + Value *thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); Type *loadT = Type::getIntNTy(ctx.builder.getContext(), nb * 8); + thePtr = emit_bitcast(ctx, thePtr, loadT->getPointerTo()); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); LoadInst *load = ctx.builder.CreateAlignedLoad(loadT, thePtr, Align(nb)); setName(ctx.emission_context, load, "atomic_pointerref"); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.decorateInst(load); load->setOrdering(llvm_order); - thePtr = strct; + thePtr = emit_bitcast(ctx, strct, thePtr->getType()); StoreInst *store = ctx.builder.CreateAlignedStore(load, thePtr, Align(julia_alignment(ety))); ai.decorateInst(store); return mark_julia_type(ctx, strct, true, ety); @@ -1047,7 +1060,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef if (!jl_isbits(ety)) { //if (!deserves_stack(ety)) - //Value *thePtr = emit_unbox(ctx, getPointerTy(ctx.builder.getContext()), e, e.typ); + //Value *thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); //uint64_t size = jl_datatype_size(ety); return emit_runtime_call(ctx, f, argv, nargs); // TODO: optimizations } @@ -1198,6 +1211,8 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ else { x_ptr = decay_derived(ctx, x_ptr); y_ptr = decay_derived(ctx, y_ptr); + if (x_ptr->getType() != y_ptr->getType()) + y_ptr = ctx.builder.CreateBitCast(y_ptr, x_ptr->getType()); ifelse_result = ctx.builder.CreateSelect(isfalse, y_ptr, x_ptr); setName(ctx.emission_context, ifelse_result, "ifelse_result"); ifelse_tbaa = MDNode::getMostGenericTBAA(x.tbaa, y.tbaa); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index c056c45337b87..b32f211c1ad8c 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2108,7 +2108,8 @@ void jl_merge_module(orc::ThreadSafeModule &destTSM, orc::ThreadSafeModule srcTS // Init.push_back(cast_or_null(Op)); // for (auto &Op : sCA->operands()) // Init.push_back(cast_or_null(Op)); - // ArrayType *ATy = ArrayType::get(PointerType::get(dest.getContext()), Init.size()); + // Type *Int8PtrTy = Type::getInt8PtrTy(dest.getContext()); + // ArrayType *ATy = ArrayType::get(Int8PtrTy, Init.size()); // GlobalVariable *GV = new GlobalVariable(dest, ATy, dG->isConstant(), // GlobalValue::AppendingLinkage, ConstantArray::get(ATy, Init), "", // dG->getThreadLocalMode(), dG->getType()->getAddressSpace()); diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index e0cde7206b6b9..c99a94c140490 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -649,7 +649,7 @@ void Optimizer::moveToStack(CallInst *orig_inst, size_t sz, bool has_ref, AllocF auto asize = ConstantInt::get(Type::getInt64Ty(prolog_builder.getContext()), sz / DL.getTypeAllocSize(pass.T_prjlvalue)); buff = prolog_builder.CreateAlloca(pass.T_prjlvalue, asize); buff->setAlignment(Align(align)); - ptr = cast(buff); + ptr = cast(prolog_builder.CreateBitCast(buff, Type::getInt8PtrTy(prolog_builder.getContext()))); } else { Type *buffty; @@ -659,14 +659,14 @@ void Optimizer::moveToStack(CallInst *orig_inst, size_t sz, bool has_ref, AllocF buffty = ArrayType::get(Type::getInt8Ty(pass.getLLVMContext()), sz); buff = prolog_builder.CreateAlloca(buffty); buff->setAlignment(Align(align)); - ptr = cast(buff); + ptr = cast(prolog_builder.CreateBitCast(buff, Type::getInt8PtrTy(prolog_builder.getContext(), buff->getType()->getPointerAddressSpace()))); } insertLifetime(ptr, ConstantInt::get(Type::getInt64Ty(prolog_builder.getContext()), sz), orig_inst); if (sz != 0 && !has_ref) { // TODO: fix has_ref case too IRBuilder<> builder(orig_inst); initializeAlloca(builder, buff, allockind); } - Instruction *new_inst = cast(ptr); + Instruction *new_inst = cast(prolog_builder.CreateBitCast(ptr, JuliaType::get_pjlvalue_ty(prolog_builder.getContext(), buff->getType()->getPointerAddressSpace()))); new_inst->takeName(orig_inst); auto simple_replace = [&] (Instruction *orig_i, Instruction *new_i) { @@ -960,7 +960,8 @@ void Optimizer::splitOnStack(CallInst *orig_inst) } slot.slot = prolog_builder.CreateAlloca(allocty); IRBuilder<> builder(orig_inst); - insertLifetime(slot.slot, ConstantInt::get(Type::getInt64Ty(prolog_builder.getContext()), field.size), orig_inst); + insertLifetime(prolog_builder.CreateBitCast(slot.slot, Type::getInt8PtrTy(prolog_builder.getContext())), + ConstantInt::get(Type::getInt64Ty(prolog_builder.getContext()), field.size), orig_inst); initializeAlloca(builder, slot.slot, use_info.allockind); slots.push_back(std::move(slot)); } @@ -1013,14 +1014,15 @@ void Optimizer::splitOnStack(CallInst *orig_inst) auto size = pass.DL->getTypeAllocSize(elty); Value *addr; if (offset % size == 0) { - addr = slot.slot; + addr = builder.CreateBitCast(slot.slot, elty->getPointerTo()); if (offset != 0) { addr = builder.CreateConstInBoundsGEP1_32(elty, addr, offset / size); } } else { - addr = slot.slot; + addr = builder.CreateBitCast(slot.slot, Type::getInt8PtrTy(builder.getContext())); addr = builder.CreateConstInBoundsGEP1_32(Type::getInt8Ty(builder.getContext()), addr, offset); + addr = builder.CreateBitCast(addr, elty->getPointerTo()); } return addr; }; @@ -1040,7 +1042,7 @@ void Optimizer::splitOnStack(CallInst *orig_inst) assert(slot.offset == offset); newload = builder.CreateLoad(pass.T_prjlvalue, slot.slot); // Assume the addrspace is correct. - val = newload; + val = builder.CreateBitCast(newload, load_ty); } else { newload = builder.CreateLoad(load_ty, slot_gep(slot, offset, load_ty, builder)); @@ -1077,6 +1079,7 @@ void Optimizer::splitOnStack(CallInst *orig_inst) } else { store_ty = PointerType::get(T_pjlvalue->getContext(), store_ty->getPointerAddressSpace()); + store_val = builder.CreateBitCast(store_val, store_ty); } if (store_ty->getPointerAddressSpace() != AddressSpace::Tracked) store_val = builder.CreateAddrSpaceCast(store_val, pass.T_prjlvalue); @@ -1141,14 +1144,14 @@ void Optimizer::splitOnStack(CallInst *orig_inst) store->setOrdering(AtomicOrdering::NotAtomic); continue; } - Value *ptr_slot = slot.slot; + auto ptr8 = builder.CreateBitCast(slot.slot, Type::getInt8PtrTy(builder.getContext())); if (offset > slot.offset) - ptr_slot = builder.CreateConstInBoundsGEP1_32(Type::getInt8Ty(builder.getContext()), slot.slot, + ptr8 = builder.CreateConstInBoundsGEP1_32(Type::getInt8Ty(builder.getContext()), ptr8, offset - slot.offset); auto sub_size = std::min(slot.offset + slot.size, offset + size) - std::max(offset, slot.offset); // TODO: alignment computation - builder.CreateMemSet(ptr_slot, val_arg, sub_size, MaybeAlign(0)); + builder.CreateMemSet(ptr8, val_arg, sub_size, MaybeAlign(0)); } call->eraseFromParent(); return; @@ -1265,8 +1268,8 @@ bool AllocOpt::doInitialization(Module &M) DL = &M.getDataLayout(); - lifetime_start = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_start, { PointerType::get(M.getContext(), DL->getAllocaAddrSpace()) }); - lifetime_end = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_end, { PointerType::get(M.getContext(), DL->getAllocaAddrSpace()) }); + lifetime_start = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_start, { Type::getInt8PtrTy(M.getContext(), DL->getAllocaAddrSpace()) }); + lifetime_end = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_end, { Type::getInt8PtrTy(M.getContext(), DL->getAllocaAddrSpace()) }); return true; } diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index 242dab021f101..e8ca62f519642 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -186,14 +186,34 @@ static inline llvm::Instruction *tbaa_decorate(llvm::MDNode *md, llvm::Instructi return inst; } +// bitcast a value, but preserve its address space when dealing with pointer types +static inline llvm::Value *emit_bitcast_with_builder(llvm::IRBuilder<> &builder, llvm::Value *v, llvm::Type *jl_value) +{ + using namespace llvm; + if (isa(jl_value) && + v->getType()->getPointerAddressSpace() != jl_value->getPointerAddressSpace()) { + // Cast to the proper address space + #if JL_LLVM_VERSION >= 170000 + Type *jl_value_addr = PointerType::get(jl_value, v->getType()->getPointerAddressSpace()); + #else + Type *jl_value_addr = PointerType::getWithSamePointeeType(cast(jl_value), v->getType()->getPointerAddressSpace()); + #endif + return builder.CreateBitCast(v, jl_value_addr); + } + else { + return builder.CreateBitCast(v, jl_value); + } +} + // Get PTLS through current task. static inline llvm::Value *get_current_task_from_pgcstack(llvm::IRBuilder<> &builder, llvm::Type *T_size, llvm::Value *pgcstack) { using namespace llvm; + auto T_ppjlvalue = JuliaType::get_ppjlvalue_ty(builder.getContext()); auto T_pjlvalue = JuliaType::get_pjlvalue_ty(builder.getContext()); const int pgcstack_offset = offsetof(jl_task_t, gcstack); return builder.CreateInBoundsGEP( - T_pjlvalue, pgcstack, + T_pjlvalue, emit_bitcast_with_builder(builder, pgcstack, T_ppjlvalue), ConstantInt::get(T_size, -(pgcstack_offset / sizeof(void *))), "current_task"); } @@ -202,17 +222,18 @@ static inline llvm::Value *get_current_task_from_pgcstack(llvm::IRBuilder<> &bui static inline llvm::Value *get_current_ptls_from_task(llvm::IRBuilder<> &builder, llvm::Type *T_size, llvm::Value *current_task, llvm::MDNode *tbaa) { using namespace llvm; + auto T_ppjlvalue = JuliaType::get_ppjlvalue_ty(builder.getContext()); auto T_pjlvalue = JuliaType::get_pjlvalue_ty(builder.getContext()); const int ptls_offset = offsetof(jl_task_t, ptls); llvm::Value *pptls = builder.CreateInBoundsGEP( - T_pjlvalue, current_task, + T_pjlvalue, emit_bitcast_with_builder(builder, current_task, T_ppjlvalue), ConstantInt::get(T_size, ptls_offset / sizeof(void *)), "ptls_field"); LoadInst *ptls_load = builder.CreateAlignedLoad(T_pjlvalue, - pptls, Align(sizeof(void *)), "ptls_load"); + emit_bitcast_with_builder(builder, pptls, T_ppjlvalue), Align(sizeof(void *)), "ptls_load"); // Note: Corresponding store (`t->ptls = ptls`) happens in `ctx_switch` of tasks.c. tbaa_decorate(tbaa, ptls_load); - return ptls_load; + return builder.CreateBitCast(ptls_load, T_ppjlvalue, "ptls"); } // Get signal page through current task. @@ -221,7 +242,9 @@ static inline llvm::Value *get_current_signal_page_from_ptls(llvm::IRBuilder<> & using namespace llvm; // return builder.CreateCall(prepare_call(reuse_signal_page_func)); auto T_psize = T_size->getPointerTo(); + auto T_ppsize = T_psize->getPointerTo(); int nthfield = offsetof(jl_tls_states_t, safepoint) / sizeof(void *); + ptls = emit_bitcast_with_builder(builder, ptls, T_ppsize); llvm::Value *psafepoint = builder.CreateInBoundsGEP( T_psize, ptls, ConstantInt::get(T_size, nthfield)); LoadInst *ptls_load = builder.CreateAlignedLoad( @@ -268,8 +291,9 @@ static inline llvm::Value *emit_gc_state_set(llvm::IRBuilder<> &builder, llvm::T { using namespace llvm; Type *T_int8 = state->getType(); + llvm::Value *ptls_i8 = emit_bitcast_with_builder(builder, ptls, builder.getInt8PtrTy()); Constant *offset = ConstantInt::getSigned(builder.getInt32Ty(), offsetof(jl_tls_states_t, gc_state)); - Value *gc_state = builder.CreateInBoundsGEP(T_int8, ptls, ArrayRef(offset), "gc_state"); + Value *gc_state = builder.CreateInBoundsGEP(T_int8, ptls_i8, ArrayRef(offset), "gc_state"); if (old_state == nullptr) { old_state = builder.CreateLoad(T_int8, gc_state); cast(old_state)->setOrdering(AtomicOrdering::Monotonic); diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index 1b0aaf505fb57..7349d5b6de927 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -103,7 +103,9 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F) IRBuilder<> builder(target); StoreInst *inst = builder.CreateAlignedStore( ConstantInt::get(T_size, JL_GC_ENCODE_PUSHARGS(nRoots)), - builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 0, "frame.nroots"),// GEP of 0 becomes a noop and eats the name + builder.CreateBitCast( + builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 0, "frame.nroots"), + T_size->getPointerTo(), "frame.nroots"), // GEP of 0 becomes a noop and eats the name Align(sizeof(void*))); inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); auto T_ppjlvalue = JuliaType::get_ppjlvalue_ty(F.getContext()); @@ -116,7 +118,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F) inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); builder.CreateAlignedStore( gcframe, - pgcstack, + builder.CreateBitCast(pgcstack, PointerType::get(PointerType::get(T_prjlvalue, 0), 0)), Align(sizeof(void*))); target->eraseFromParent(); } @@ -134,7 +136,8 @@ void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F) inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); inst = builder.CreateAlignedStore( inst, - pgcstack, + builder.CreateBitCast(pgcstack, + PointerType::get(T_prjlvalue, 0)), Align(sizeof(void*))); inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); target->eraseFromParent(); diff --git a/src/llvm-julia-licm.cpp b/src/llvm-julia-licm.cpp index 8d80f7fd54877..e76beaa3df44f 100644 --- a/src/llvm-julia-licm.cpp +++ b/src/llvm-julia-licm.cpp @@ -330,9 +330,10 @@ struct JuliaLICM : public JuliaPassContext { moveInstructionBefore(*call, *preheader->getTerminator(), MSSAU, SE); IRBuilder<> builder(preheader->getTerminator()); builder.SetCurrentDebugLocation(call->getDebugLoc()); + auto obj_i8 = builder.CreateBitCast(call, Type::getInt8PtrTy(call->getContext(), call->getType()->getPointerAddressSpace())); // Note that this alignment is assuming the GC allocates at least pointer-aligned memory auto align = Align(DL.getPointerSize(0)); - auto clear_obj = builder.CreateMemSet(call, ConstantInt::get(Type::getInt8Ty(call->getContext()), 0), call->getArgOperand(1), align); + auto clear_obj = builder.CreateMemSet(obj_i8, ConstantInt::get(Type::getInt8Ty(call->getContext()), 0), call->getArgOperand(1), align); if (MSSAU.getMemorySSA()) { auto clear_mdef = MSSAU.createMemoryAccessInBB(clear_obj, nullptr, clear_obj->getParent(), MemorySSA::BeforeTerminator); MSSAU.insertDef(cast(clear_mdef), true); diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 59886a134fd4e..55efa37fccf0b 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -3,7 +3,6 @@ #include "llvm-version.h" #include "passes.h" -#include "llvm/IR/DerivedTypes.h" #include #include @@ -511,7 +510,7 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac // This could really be anything, but it's not loaded // from a tracked pointer, so it doesn't matter what // it is--just pick something simple. - CurrentV = ConstantPointerNull::get(PointerType::get(V->getContext(), 0)); + CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); } continue; } @@ -546,12 +545,12 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac if (II->getIntrinsicID() == Intrinsic::masked_load) { fld_idx = -1; if (!isSpecialPtr(CurrentV->getType())) { - CurrentV = ConstantPointerNull::get(PointerType::get(V->getContext(), 0)); + CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); } } else { if (auto VTy2 = dyn_cast(CurrentV->getType())) { if (!isSpecialPtr(VTy2->getElementType())) { - CurrentV = ConstantPointerNull::get(PointerType::get(V->getContext(), 0)); + CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); fld_idx = -1; } } @@ -2257,7 +2256,9 @@ SmallVector LateLowerGCFrame::ColorRoots(const State &S) { Value *LateLowerGCFrame::EmitTagPtr(IRBuilder<> &builder, Type *T, Type *T_size, Value *V) { assert(T == T_size || isa(T)); - return builder.CreateInBoundsGEP(T, V, ConstantInt::get(T_size, -1), V->getName() + ".tag_addr"); + auto TV = cast(V->getType()); + auto cast = builder.CreateBitCast(V, T->getPointerTo(TV->getAddressSpace())); + return builder.CreateInBoundsGEP(T, cast, ConstantInt::get(T_size, -1), V->getName() + ".tag_addr"); } Value *LateLowerGCFrame::EmitLoadTag(IRBuilder<> &builder, Type *T_size, Value *V) @@ -2437,7 +2438,8 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { // the type tag. (Note that if the size is not a constant, it will call // gc_alloc_obj, and will redundantly set the tag.) auto allocBytesIntrinsic = getOrDeclare(jl_intrinsics::GCAllocBytes); - auto ptls = get_current_ptls_from_task(builder, T_size, CI->getArgOperand(0), tbaa_gcframe); + auto ptlsLoad = get_current_ptls_from_task(builder, T_size, CI->getArgOperand(0), tbaa_gcframe); + auto ptls = builder.CreateBitCast(ptlsLoad, Type::getInt8PtrTy(builder.getContext())); auto newI = builder.CreateCall( allocBytesIntrinsic, { @@ -2518,7 +2520,7 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { // the julia.call signature is varargs, the optimizer is allowed // to rewrite pointee types. It'll go away with opaque pointer // types anyway. - Builder.CreateAlignedStore(*arg_it, + Builder.CreateAlignedStore(Builder.CreateBitCast(*arg_it, T_prjlvalue), Builder.CreateInBoundsGEP(T_prjlvalue, Frame, ConstantInt::get(T_int32, slot++)), Align(sizeof(void*))); } diff --git a/src/llvm-lower-handlers.cpp b/src/llvm-lower-handlers.cpp index e09ea892ee488..35ce322ad3a71 100644 --- a/src/llvm-lower-handlers.cpp +++ b/src/llvm-lower-handlers.cpp @@ -88,15 +88,16 @@ namespace { */ static void ensure_enter_function(Module &M, Type *T_pjlvalue, const Triple &TT) { - auto T_ptr = PointerType::get(M.getContext(), 0); + auto T_int8 = Type::getInt8Ty(M.getContext()); + auto T_pint8 = PointerType::get(T_int8, 0); auto T_void = Type::getVoidTy(M.getContext()); auto T_int32 = Type::getInt32Ty(M.getContext()); if (!M.getNamedValue(XSTR(jl_enter_handler))) { - Function::Create(FunctionType::get(T_void, {T_pjlvalue, T_ptr}, false), + Function::Create(FunctionType::get(T_void, {T_pjlvalue, T_pint8}, false), Function::ExternalLinkage, XSTR(jl_enter_handler), &M); } if (!M.getNamedValue(jl_setjmp_name)) { - Type *args2[] = {T_ptr, T_int32}; + Type *args2[] = {T_pint8, T_int32}; Function::Create(FunctionType::get(T_int32, ArrayRef(args2, TT.isOSWindows() ? 1 : 2), false), Function::ExternalLinkage, jl_setjmp_name, &M) ->addFnAttr(Attribute::ReturnsTwice); @@ -114,9 +115,10 @@ static bool lowerExcHandlers(Function &F) { Function *leave_noexcept_func = M.getFunction(XSTR(jl_pop_handler_noexcept)); Function *jlenter_func = M.getFunction(XSTR(jl_enter_handler)); Function *setjmp_func = M.getFunction(jl_setjmp_name); - auto T_ptr = PointerType::get(M.getContext(), 0); - Function *lifetime_start = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_start, { T_ptr }); - Function *lifetime_end = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_end, { T_ptr }); + + auto T_pint8 = Type::getInt8PtrTy(M.getContext(), 0); + Function *lifetime_start = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_start, { T_pint8 }); + Function *lifetime_end = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_end, { T_pint8 }); /* Step 1: EH Depth Numbering */ std::map EnterDepth; @@ -177,7 +179,7 @@ static bool lowerExcHandlers(Function &F) { auto *buff = new AllocaInst(Type::getInt8Ty(F.getContext()), allocaAddressSpace, handler_sz, Align(16), "", firstInst); if (allocaAddressSpace) { - AddrSpaceCastInst *buff_casted = new AddrSpaceCastInst(buff, PointerType::get(F.getContext(), AddressSpace::Generic)); + AddrSpaceCastInst *buff_casted = new AddrSpaceCastInst(buff, Type::getInt8PtrTy(F.getContext(), AddressSpace::Generic)); buff_casted->insertAfter(buff); buffs.push_back(buff_casted); } else { diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 0f498973671ec..00a9c08576a6c 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -8,7 +8,6 @@ #include "llvm-version.h" #include "llvm/IR/Attributes.h" -#include "llvm/IR/DerivedTypes.h" #include #include #include @@ -163,7 +162,7 @@ namespace jl_intrinsics { auto intrinsic = Function::Create( FunctionType::get( T_prjlvalue, - { PointerType::get(ctx, 0), + { Type::getInt8PtrTy(ctx), T_size, T_size }, // type false), @@ -273,7 +272,7 @@ namespace jl_well_known { auto bigAllocFunc = Function::Create( FunctionType::get( T_prjlvalue, - { PointerType::get(ctx, 0), T_size , T_size}, + { Type::getInt8PtrTy(ctx), T_size , T_size}, false), Function::ExternalLinkage, GC_BIG_ALLOC_NAME); @@ -289,7 +288,7 @@ namespace jl_well_known { auto poolAllocFunc = Function::Create( FunctionType::get( T_prjlvalue, - { PointerType::get(ctx, 0), Type::getInt32Ty(ctx), Type::getInt32Ty(ctx), T_size }, + { Type::getInt8PtrTy(ctx), Type::getInt32Ty(ctx), Type::getInt32Ty(ctx), T_size }, false), Function::ExternalLinkage, GC_POOL_ALLOC_NAME); @@ -325,7 +324,7 @@ namespace jl_well_known { auto allocTypedFunc = Function::Create( FunctionType::get( T_prjlvalue, - { PointerType::get(ctx, 0), + { Type::getInt8PtrTy(ctx), T_size, T_size }, // type false), diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 9e49aa5ba2f39..b2f58ca79f1c3 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -95,12 +95,12 @@ Instruction *LowerPTLS::emit_pgcstack_tp(Value *offset, Instruction *insertBefor if (offset) { SmallVector args(0); args.push_back(offset->getType()); - auto tp = InlineAsm::get(FunctionType::get(PointerType::get(builder.getContext(), 0), args, false), + auto tp = InlineAsm::get(FunctionType::get(Type::getInt8PtrTy(builder.getContext()), args, false), dyn_asm_str, "=&r,r,~{dirflag},~{fpsr},~{flags}", false); tls = builder.CreateCall(tp, {offset}, "pgcstack"); } else { - auto tp = InlineAsm::get(FunctionType::get(PointerType::get(builder.getContext(), 0), false), + auto tp = InlineAsm::get(FunctionType::get(Type::getInt8PtrTy(insertBefore->getContext()), false), const_asm_str.c_str(), "=r,~{dirflag},~{fpsr},~{flags}", false); tls = builder.CreateCall(tp, {}, "tls_pgcstack"); @@ -126,10 +126,11 @@ Instruction *LowerPTLS::emit_pgcstack_tp(Value *offset, Instruction *insertBefor } if (!offset) offset = ConstantInt::getSigned(T_size, jl_tls_offset); - auto tp = InlineAsm::get(FunctionType::get(PointerType::get(builder.getContext(), 0), false), asm_str, "=r", false); + auto tp = InlineAsm::get(FunctionType::get(Type::getInt8PtrTy(builder.getContext()), false), asm_str, "=r", false); tls = builder.CreateCall(tp, {}, "thread_ptr"); tls = builder.CreateGEP(Type::getInt8Ty(builder.getContext()), tls, {offset}, "tls_ppgcstack"); } + tls = builder.CreateBitCast(tls, T_pppjlvalue->getPointerTo()); return builder.CreateLoad(T_pppjlvalue, tls, "tls_pgcstack"); } diff --git a/src/llvm-remove-addrspaces.cpp b/src/llvm-remove-addrspaces.cpp index 89ae1d292d108..dc9642ca1e81e 100644 --- a/src/llvm-remove-addrspaces.cpp +++ b/src/llvm-remove-addrspaces.cpp @@ -220,8 +220,10 @@ bool RemoveNoopAddrSpaceCasts(Function *F) if (ASC->getType() == ASC->getOperand(0)->getType()) { ASC->replaceAllUsesWith(ASC->getOperand(0)); } else { - // uncanonicalized addrspacecast; just use the value - ASC->replaceAllUsesWith(ASC->getOperand(0)); + // uncanonicalized addrspacecast; demote to bitcast + llvm::IRBuilder<> builder(ASC); + auto BC = builder.CreateBitCast(ASC->getOperand(0), ASC->getType()); + ASC->replaceAllUsesWith(BC); } NoopCasts.push_back(ASC); }