Skip to content

Commit

Permalink
codegen: NFC refactoring to use Align type
Browse files Browse the repository at this point in the history
  • Loading branch information
@vtjnash
vtjnash committed Aug 3, 2024
1 parent f2f188d commit f38015f
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 39 deletions.
10 changes: 5 additions & 5 deletions src/ccall.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,8 +554,8 @@ static Value *julia_to_native(
// pass the address of an alloca'd thing, not a box
// since those are immutable.
Value *slot = emit_static_alloca(ctx, to);
unsigned align = julia_alignment(jlto);
cast<AllocaInst>(slot)->setAlignment(Align(align));
Align align(julia_alignment(jlto));
cast<AllocaInst>(slot)->setAlignment(align);
setName(ctx.emission_context, slot, "native_convert_buffer");
if (!jvinfo.ispointer()) {
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, jvinfo.tbaa);
Expand Down Expand Up @@ -2230,7 +2230,7 @@ jl_cgval_t function_sig_t::emit_a_ccall(
Value *strct = emit_allocobj(ctx, (jl_datatype_t*)rt, true);
setName(ctx.emission_context, strct, "ccall_ret_box");
MDNode *tbaa = jl_is_mutable(rt) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut;
int boxalign = julia_alignment(rt);
Align boxalign(julia_alignment(rt));
// copy the data from the return value to the new struct
const DataLayout &DL = ctx.builder.GetInsertBlock()->getModule()->getDataLayout();
auto resultTy = result->getType();
Expand All @@ -2240,8 +2240,8 @@ jl_cgval_t function_sig_t::emit_a_ccall(
// When this happens, cast through memory.
auto slot = emit_static_alloca(ctx, resultTy);
setName(ctx.emission_context, slot, "type_pun_slot");
slot->setAlignment(Align(boxalign));
ctx.builder.CreateAlignedStore(result, slot, Align(boxalign));
slot->setAlignment(boxalign);
ctx.builder.CreateAlignedStore(result, slot, boxalign);
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
emit_memcpy(ctx, strct, ai, slot, ai, rtsz, boxalign, boxalign);
}
Expand Down
41 changes: 20 additions & 21 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V)
}

static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt);
static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile=false);
static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, Align alignment, bool isVolatile=false);

static bool type_is_permalloc(jl_value_t *typ)
{
Expand Down Expand Up @@ -1006,11 +1006,10 @@ static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x)
}

static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src,
jl_aliasinfo_t const &src_ai, uint64_t sz, unsigned align_dst, unsigned align_src, bool is_volatile)
jl_aliasinfo_t const &src_ai, uint64_t sz, Align align_dst, Align align_src, bool is_volatile)
{
if (sz == 0)
return;
assert(align_dst && "align must be specified");
#if JL_LLVM_VERSION < 170000
// If the types are small and simple, use load and store directly.
// Going through memcpy can cause LLVM (e.g. SROA) to create bitcasts between float and int
Expand Down Expand Up @@ -1053,7 +1052,7 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const
if (isa<Instruction>(dst) && !dst->hasName())
setName(ctx.emission_context, dst, "memcpy_refined_dst");
auto val = src_ai.decorateInst(ctx.builder.CreateAlignedLoad(directel, src, MaybeAlign(align_src), is_volatile));
dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, Align(align_dst), is_volatile));
dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, align_dst, is_volatile));
++SkippedMemcpys;
return;
}
Expand All @@ -1072,12 +1071,12 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const
// above problem won't be as serious.

auto merged_ai = dst_ai.merge(src_ai);
ctx.builder.CreateMemCpy(dst, Align(align_dst), src, Align(align_src), sz, is_volatile,
ctx.builder.CreateMemCpy(dst, align_dst, src, align_src, sz, is_volatile,
merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias);
}

static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src,
jl_aliasinfo_t const &src_ai, Value *sz, unsigned align_dst, unsigned align_src, bool is_volatile)
jl_aliasinfo_t const &src_ai, Value *sz, Align align_dst, Align align_src, bool is_volatile)
{
if (auto const_sz = dyn_cast<ConstantInt>(sz)) {
emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, const_sz->getZExtValue(), align_dst, align_src, is_volatile);
Expand All @@ -1086,20 +1085,20 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const
++EmittedMemcpys;

auto merged_ai = dst_ai.merge(src_ai);
ctx.builder.CreateMemCpy(dst, MaybeAlign(align_dst), src, MaybeAlign(align_src), sz, is_volatile,
ctx.builder.CreateMemCpy(dst, align_dst, src, align_src, sz, is_volatile,
merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias);
}

template<typename T1>
static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src,
jl_aliasinfo_t const &src_ai, T1 &&sz, unsigned align_dst, unsigned align_src, bool is_volatile=false)
jl_aliasinfo_t const &src_ai, T1 &&sz, Align align_dst, Align align_src, bool is_volatile=false)
{
emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, sz, align_dst, align_src, is_volatile);
}

template<typename T1>
static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, const jl_cgval_t &src,
T1 &&sz, unsigned align_dst, unsigned align_src, bool is_volatile=false)
T1 &&sz, Align align_dst, Align align_src, bool is_volatile=false)
{
auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, src.tbaa);
emit_memcpy_llvm(ctx, dst, dst_ai, data_pointer(ctx, src), src_ai, sz, align_dst, align_src, is_volatile);
Expand Down Expand Up @@ -1999,7 +1998,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
else if (!alignment)
alignment = julia_alignment(jltype);
if (intcast && Order == AtomicOrdering::NotAtomic) {
emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, alignment, intcast->getAlign().value());
emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, Align(alignment), intcast->getAlign());
}
else {
if (!isboxed && jl_is_genericmemoryref_type(jltype)) {
Expand Down Expand Up @@ -2176,7 +2175,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
}
else {
assert(Order == AtomicOrdering::NotAtomic && !isboxed && rhs.typ == jltype);
emit_unbox_store(ctx, rhs, ptr, tbaa, alignment);
emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment));
}
}
else if (isswapfield) {
Expand Down Expand Up @@ -2329,7 +2328,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
}
else {
assert(!isboxed && rhs.typ == jltype);
emit_unbox_store(ctx, rhs, ptr, tbaa, alignment);
emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment));
}
ctx.builder.CreateBr(DoneBB);
instr = load;
Expand Down Expand Up @@ -2668,7 +2667,7 @@ static jl_cgval_t emit_unionload(jl_codectx_t &ctx, Value *addr, Value *ptindex,
if (al > 1)
lv->setAlignment(Align(al));
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
emit_memcpy(ctx, lv, ai, addr, ai, fsz, al, al);
emit_memcpy(ctx, lv, ai, addr, ai, fsz, Align(al), Align(al));
addr = lv;
}
return mark_julia_slot(fsz > 0 ? addr : nullptr, jfty, tindex, tbaa);
Expand Down Expand Up @@ -3104,19 +3103,19 @@ static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t)
static Value *emit_allocobj(jl_codectx_t &ctx, jl_datatype_t *jt, bool fully_initialized);

static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tbaa,
unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned
Align alignment = Align(sizeof(void*))) // min alignment in julia's gc is pointer-aligned
{
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
// newv should already be tagged
ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, Align(alignment)));
ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, alignment));
}

static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, MDNode *tbaa)
{
// newv should already be tagged
if (v.ispointer()) {
unsigned align = std::max(julia_alignment(v.typ), (unsigned)sizeof(void*));
emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), align, julia_alignment(v.typ));
emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), Align(align), Align(julia_alignment(v.typ)));
}
else {
init_bits_value(ctx, newv, v.V, tbaa);
Expand Down Expand Up @@ -3582,7 +3581,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
if (jl_is_pointerfree(typ)) {
unsigned alignment = julia_alignment(typ);
if (!src.ispointer() || src.constant) {
emit_unbox_store(ctx, src, dest, tbaa_dst, alignment, isVolatile);
emit_unbox_store(ctx, src, dest, tbaa_dst, Align(alignment), isVolatile);
}
else {
Value *src_ptr = data_pointer(ctx, src);
Expand All @@ -3592,7 +3591,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
// if (skip) src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr);
auto f = [&] {
(void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr,
jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile);
jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, Align(alignment), Align(alignment), isVolatile);
return nullptr;
};
if (skip)
Expand Down Expand Up @@ -3627,7 +3626,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
return;
} else {
emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr,
jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile);
jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, Align(alignment), Align(alignment), isVolatile);
}
}
ctx.builder.CreateBr(postBB);
Expand All @@ -3653,7 +3652,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
Value *datatype = emit_typeof(ctx, src, false, false);
Value *copy_bytes = emit_datatype_size(ctx, datatype);
(void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src),
jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, 1, 1, isVolatile);
jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, Align(1), Align(1), isVolatile);
return nullptr;
};
if (skip)
Expand Down Expand Up @@ -4046,7 +4045,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
} else if (init_as_value) {
fval = emit_unbox(ctx, fty, fval_info, jtype);
} else {
emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, jl_field_align(sty, i));
emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, Align(jl_field_align(sty, i)));
}
}
if (init_as_value) {
Expand Down
17 changes: 10 additions & 7 deletions src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5652,7 +5652,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va
else {
const DataLayout &DL = jl_Module->getDataLayout();
uint64_t sz = DL.getTypeStoreSize(T);
emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign().value(), varslot->getAlign().value());
emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign(), varslot->getAlign());
}
Value *tindex = NULL;
if (vi.pTIndex)
Expand Down Expand Up @@ -5756,8 +5756,9 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu
// This check should probably mostly catch the relevant situations.
if (vi.value.V != rval_info.V) {
Value *copy_bytes = ConstantInt::get(getInt32Ty(ctx.builder.getContext()), jl_datatype_size(vi.value.typ));
Align alignment(julia_alignment(rval_info.typ));
emit_memcpy(ctx, vi.value.V, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), rval_info, copy_bytes,
julia_alignment(rval_info.typ), julia_alignment(rval_info.typ), vi.isVolatile);
alignment, alignment, vi.isVolatile);
}
}
else {
Expand Down Expand Up @@ -6868,8 +6869,9 @@ static void emit_cfunc_invalidate(
root1 = ctx.builder.CreateConstInBoundsGEP2_32(get_returnroots_type(ctx, return_roots), root1, 0, 0);
ctx.builder.CreateStore(gf_ret, root1);
}
Align alignment(julia_alignment(rettype));
emit_memcpy(ctx, &*gf_thunk->arg_begin(), jl_aliasinfo_t::fromTBAA(ctx, nullptr), gf_ret,
jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), julia_alignment(rettype), julia_alignment(rettype));
jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), Align(alignment), Align(alignment));
ctx.builder.CreateRetVoid();
break;
}
Expand Down Expand Up @@ -8729,7 +8731,7 @@ static jl_llvm_functions_t
jl_cgval_t closure_world = typed_load(ctx, worldaddr, NULL, (jl_value_t*)jl_long_type,
nullptr, nullptr, false, AtomicOrdering::NotAtomic, false, ctx.types().alignof_ptr.value());
ctx.world_age_at_entry = closure_world.V; // The tls world in a OC is the world of the closure
emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr.value());
emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr);

// Load closure env
Value *envaddr = ctx.builder.CreateInBoundsGEP(
Expand Down Expand Up @@ -9272,8 +9274,9 @@ static jl_llvm_functions_t
}
if (returninfo.cc == jl_returninfo_t::SRet) {
assert(jl_is_concrete_type(jlrettype));
Align alignment(julia_alignment(jlrettype));
emit_memcpy(ctx, sret, jl_aliasinfo_t::fromTBAA(ctx, nullptr), retvalinfo,
jl_datatype_size(jlrettype), julia_alignment(jlrettype), julia_alignment(jlrettype));
jl_datatype_size(jlrettype), alignment, alignment);
}
else { // must be jl_returninfo_t::Union
emit_unionmove(ctx, sret, nullptr, retvalinfo, /*skip*/isboxed_union);
Expand Down Expand Up @@ -9511,7 +9514,7 @@ static jl_llvm_functions_t
// load of val) if the runtime type of val isn't phiType
Value *isvalid = emit_isa_and_defined(ctx, val, phiType);
emit_guarded_test(ctx, isvalid, nullptr, [&] {
emit_unbox_store(ctx, update_julia_type(ctx, val, phiType), dest, ctx.tbaa().tbaa_stack, julia_alignment(phiType));
emit_unbox_store(ctx, update_julia_type(ctx, val, phiType), dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(phiType)));
return nullptr;
});
}
Expand All @@ -9538,7 +9541,7 @@ static jl_llvm_functions_t
if (VN)
V = Constant::getNullValue(ctx.types().T_prjlvalue);
if (dest)
emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, julia_alignment(val.typ));
emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(val.typ)));
RTindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), tindex);
}
}
Expand Down
Loading

0 comments on commit f38015f

Please sign in to comment.