Skip to content

Commit

Permalink
codegen: many sizeof computation fixes
Browse files Browse the repository at this point in the history 
jl_datatype_size is rounded up to the allocation size
however, many allocations in llvm are instead shrink-wrapped to the actual size of the data
  • Loading branch information
@vtjnash
vtjnash committed Oct 13, 2017
1 parent 39f668c commit 05bc1b7
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 87 deletions.
165 changes: 98 additions & 67 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -895,6 +895,69 @@ static Value *emit_datatype_name(jl_codectx_t &ctx, Value *dt)
tbaa_const);
}

// If given alignment is 0 and LLVM's assumed alignment for a load/store via ptr
// might be stricter than the Julia alignment for jltype, return the alignment of jltype.
// Otherwise return the given alignment.
static unsigned llvm_alignment(jl_value_t *jltype, unsigned alignment)
{
if (!alignment) {
alignment = jl_datatype_align(jltype);
assert(alignment <= JL_HEAP_ALIGNMENT);
assert(JL_HEAP_ALIGNMENT % alignment == 0);
}
return alignment;
}

static unsigned llvm_sizeof(jl_datatype_t *dt)
{
unsigned nf = jl_datatype_nfields(dt);
if (nf == 0)
return jl_datatype_size(dt);
return jl_field_offset(dt, nf - 1)
+ jl_field_size(dt, nf - 1);
}

static unsigned llvm_sizeof(jl_value_t *dt)
{
return llvm_sizeof((jl_datatype_t*)dt);
}

static size_t dereferenceable_size(jl_value_t *jt, bool isfield) {
size_t size = 0;
if (jl_is_array_type(jt)) {
// Array has at least this much data
size = sizeof(jl_array_t);
}
else if (isfield) {
size = jl_datatype_size(jt);
}
else {
size = llvm_sizeof(jt);
}
return size;
}

static inline Instruction *maybe_mark_load_dereferenceable(Instruction *LI, bool can_be_null, size_t size) {
if (!size) {
return LI;
}
llvm::SmallVector<Metadata *, 1> OPs;
OPs.push_back(ConstantAsMetadata::get(ConstantInt::get(T_int64, size)));
LI->setMetadata(can_be_null ? "dereferenceable_or_null" :
"dereferenceable",
MDNode::get(jl_LLVMContext, OPs));
return LI;
}

static inline Instruction *maybe_mark_load_dereferenceable(Instruction *LI, bool can_be_null, jl_value_t *jt, bool isfield) {
if (!jl_is_leaf_type(jt)) {
return LI;
}
size_t size = dereferenceable_size(jt, isfield);
return maybe_mark_load_dereferenceable(LI, can_be_null, size);
}


// --- generating various error checks ---
// Do not use conditional throw for cases that type inference can know
// the error is always thrown. This may cause non dominated use
Expand Down Expand Up @@ -956,48 +1019,6 @@ static void raise_exception_unless(jl_codectx_t &ctx, Value *cond, Value *exc)
raise_exception(ctx, exc, passBB);
}

static size_t dereferenceable_size(jl_value_t *jt) {
size_t size = 0;
if (jl_is_array_type(jt)) {
// Array has at least this much data
size = sizeof(jl_array_t);
} else {
size = jl_datatype_size(jt);
}
return size;
}

static inline void maybe_mark_argument_dereferenceable(Argument *A, jl_value_t *jt) {
if (!jl_is_leaf_type(jt)) {
return;
}
size_t size = dereferenceable_size(jt);
if (!size) {
return;
}
A->getParent()->addDereferenceableAttr(A->getArgNo() + 1, size);
}

static inline Instruction *maybe_mark_load_dereferenceable(Instruction *LI, bool can_be_null, size_t size) {
if (!size) {
return LI;
}
llvm::SmallVector<Metadata *, 1> OPs;
OPs.push_back(ConstantAsMetadata::get(ConstantInt::get(T_int64, size)));
LI->setMetadata(can_be_null ? "dereferenceable_or_null" :
"dereferenceable",
MDNode::get(jl_LLVMContext, OPs));
return LI;
}

static inline Instruction *maybe_mark_load_dereferenceable(Instruction *LI, bool can_be_null, jl_value_t *jt) {
if (!jl_is_leaf_type(jt)) {
return LI;
}
size_t size = dereferenceable_size(jt);
return maybe_mark_load_dereferenceable(LI, can_be_null, size);
}

static void null_pointer_check(jl_codectx_t &ctx, Value *v)
{
raise_exception_unless(ctx,
Expand Down Expand Up @@ -1171,18 +1192,6 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v
return im1;
}

// If given alignment is 0 and LLVM's assumed alignment for a load/store via ptr
// might be stricter than the Julia alignment for jltype, return the alignment of jltype.
// Otherwise return the given alignment.
static unsigned julia_alignment(jl_value_t *jltype, unsigned alignment)
{
if (!alignment) {
alignment = jl_datatype_align(jltype);
assert(alignment <= JL_HEAP_ALIGNMENT);
assert(JL_HEAP_ALIGNMENT % alignment == 0);
}
return alignment;
}

static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt, Value* dest = NULL, bool volatile_store = false);

Expand Down Expand Up @@ -1210,9 +1219,9 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
//}
//else {
Instruction *load = ctx.builder.CreateAlignedLoad(data, isboxed ?
alignment : julia_alignment(jltype, alignment), false);
alignment : llvm_alignment(jltype, alignment), false);
if (isboxed)
load = maybe_mark_load_dereferenceable(load, true, jltype);
load = maybe_mark_load_dereferenceable(load, true, jltype, false);
if (tbaa) {
elt = tbaa_decorate(tbaa, load);
}
Expand Down Expand Up @@ -1257,7 +1266,7 @@ static void typed_store(jl_codectx_t &ctx,
data = ptr;
}
Instruction *store = ctx.builder.CreateAlignedStore(r, idx_0based ? ctx.builder.CreateGEP(data,
idx_0based) : data, isboxed ? alignment : julia_alignment(jltype, alignment));
idx_0based) : data, isboxed ? alignment : llvm_alignment(jltype, alignment));
if (tbaa)
tbaa_decorate(tbaa, store);
}
Expand Down Expand Up @@ -1369,7 +1378,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx,
size_t minimum_field_size = (size_t)-1;
for (size_t i = 0; i < nfields; ++i) {
minimum_field_size = std::min(minimum_field_size,
dereferenceable_size(jl_field_type(stt, i)));
dereferenceable_size(jl_field_type(stt, i), true));
if (minimum_field_size == 0)
break;
}
Expand All @@ -1380,7 +1389,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx,
ctx.builder.CreateGEP(decay_derived(
emit_bitcast(ctx, data_pointer(ctx, strct), T_pprjlvalue)), idx),
PointerType::get(T_prjlvalue, AddressSpace::Derived))),
maybe_null, minimum_field_size));
maybe_null, minimum_field_size));
if (maybe_null)
null_pointer_check(ctx, fld);
*ret = mark_julia_type(ctx, fld, true, jl_any_type);
Expand Down Expand Up @@ -1483,8 +1492,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
bool maybe_null = idx >= (unsigned)jt->ninitialized;
Instruction *Load = maybe_mark_load_dereferenceable(
ctx.builder.CreateLoad(T_prjlvalue, emit_bitcast(ctx, addr, T_pprjlvalue)),
maybe_null, jl_field_type(jt, idx)
);
maybe_null, jl_field_type(jt, idx), true);
Value *fldv = tbaa_decorate(strct.tbaa, Load);
if (maybe_null)
null_pointer_check(ctx, fldv);
Expand Down Expand Up @@ -1850,7 +1858,7 @@ static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v,
{
// newv should already be tagged
if (v.ispointer()) {
emit_memcpy(ctx, newv, v, jl_datatype_size(v.typ), sizeof(void*));
emit_memcpy(ctx, newv, v, llvm_sizeof(v.typ), sizeof(void*));
}
else {
init_bits_value(ctx, newv, v.V, tbaa);
Expand Down Expand Up @@ -2165,13 +2173,14 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, const jl_cgval_t &src
}
else {
Value *src_ptr = data_pointer(ctx, src);
unsigned nb = jl_datatype_size(typ);
unsigned alignment = julia_alignment(typ, 0);
unsigned nb = llvm_sizeof(typ);
unsigned alignment = llvm_alignment(typ, 0);
Value *nbytes = ConstantInt::get(T_size, nb);
if (skip) // copy dest -> dest to simulate an undef value / conditional copy
if (skip) { // TODO: this is VERY expensive
nbytes = ctx.builder.CreateSelect(skip,
ConstantInt::get(T_size, 0),
nbytes);
}
emit_memcpy(ctx, dest, src_ptr, nbytes, alignment, isVolatile, tbaa);
}
}
Expand All @@ -2188,8 +2197,8 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, const jl_cgval_t &src
unsigned counter = 0;
bool allunboxed = for_each_uniontype_small(
[&](unsigned idx, jl_datatype_t *jt) {
unsigned nb = jl_datatype_size(jt);
unsigned alignment = julia_alignment((jl_value_t*)jt, 0);
unsigned nb = llvm_sizeof(jt);
unsigned alignment = llvm_alignment((jl_value_t*)jt, 0);
BasicBlock *tempBB = BasicBlock::Create(jl_LLVMContext, "union_move", ctx.f);
ctx.builder.SetInsertPoint(tempBB);
switchInst->addCase(ConstantInt::get(T_int8, idx), tempBB);
Expand All @@ -2213,8 +2222,30 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, const jl_cgval_t &src
ctx.builder.SetInsertPoint(postBB);
}
else {
assert(src.isboxed && "expected boxed value for sizeof/alignment computation");
Value *datatype = emit_typeof_boxed(ctx, src);
Value *copy_bytes = emit_datatype_size(ctx, datatype);
Value *max_bytes = NULL;
// In some cases, we know the destination isn't large enough to hold the whole value.
// Currently, we can detect those cases precisely with a isa test.
if (auto AI = dyn_cast<AllocaInst>(dest)) {
const DataLayout &DL =
#if JL_LLVM_VERSION >= 40000
jl_data_layout;
#else
jl_ExecutionEngine->getDataLayout();
#endif
max_bytes = ConstantInt::get(copy_bytes->getType(), DL.getTypeStoreSize(AI->getAllocatedType()));
if (AI->isArrayAllocation())
max_bytes = ctx.builder.CreateNUWMul(max_bytes, AI->getArraySize());
}
if (auto AI = dyn_cast<Argument>(dest)) {
size_t bytes = AI->getDereferenceableBytes();
assert(bytes > 0);
max_bytes = ConstantInt::get(copy_bytes->getType(), bytes);
}
if (max_bytes)
copy_bytes = ctx.builder.CreateSelect(ctx.builder.CreateICmpULT(copy_bytes, max_bytes), copy_bytes, max_bytes);
if (skip)
copy_bytes = ctx.builder.CreateSelect(skip, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes);
emit_memcpy(ctx, dest, src, copy_bytes, /*TODO: min-align*/1);
Expand Down
40 changes: 22 additions & 18 deletions src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3436,12 +3436,12 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu
assert(jl_is_leaf_type(vi.value.typ));
// Sometimes we can get into situations where the LHS and RHS
// are the same slot. We're not allowed to memcpy in that case
// under penalty of undefined behavior. This check should catch
// the relevant situations.
// under penalty of undefined behavior.
// This check should probably mostly catch the relevant situations.
if (vi.value.V != rval_info.V) {
Value *copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(vi.value.typ));
Value *copy_bytes = ConstantInt::get(T_int32, llvm_sizeof(vi.value.typ));
emit_memcpy(ctx, vi.value.V, rval_info, copy_bytes,
jl_datatype_align(rval_info.typ), vi.isVolatile, tbaa);
llvm_alignment(rval_info.typ, 0), vi.isVolatile, tbaa);
}
}
else {
Expand Down Expand Up @@ -4057,12 +4057,12 @@ static void emit_cfunc_invalidate(
}
else {
gf_ret = emit_bitcast(ctx, gf_ret, gfrt->getPointerTo());
ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gf_ret, julia_alignment(astrt, 0)));
ctx.builder.CreateRet(ctx.builder.CreateAlignedLoad(gf_ret, llvm_alignment(astrt, 0)));
}
break;
}
case jl_returninfo_t::SRet: {
unsigned sret_nbytes = jl_datatype_size(astrt);
unsigned sret_nbytes = llvm_sizeof(astrt);
emit_memcpy(ctx, &*gf_thunk->arg_begin(), gf_ret, sret_nbytes, jl_alignment(sret_nbytes));
ctx.builder.CreateRetVoid();
break;
Expand Down Expand Up @@ -4606,7 +4606,7 @@ static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returnin
if (lty != NULL && !isboxed) {
theArg = decay_derived(emit_bitcast(ctx, theArg, PointerType::get(lty, 0)));
if (!lty->isAggregateType()) // keep "aggregate" type values in place as pointers
theArg = ctx.builder.CreateAlignedLoad(theArg, julia_alignment(ty, 0));
theArg = ctx.builder.CreateAlignedLoad(theArg, llvm_alignment(ty, 0));
}
assert(dyn_cast<UndefValue>(theArg) == NULL);
args[idx] = theArg;
Expand Down Expand Up @@ -4718,7 +4718,7 @@ static jl_returninfo_t get_specsig_function(Module *M, const std::string &name,
bool retboxed;
rt = julia_type_to_llvm(jlrettype, &retboxed);
if (!retboxed) {
if (rt != T_void && deserves_sret(jlrettype, rt)) {
if (rt != T_void && llvm_sizeof(jlrettype) > 0 && deserves_sret(jlrettype, rt)) {
props.cc = jl_returninfo_t::SRet;
fsig.push_back(rt->getPointerTo(AddressSpace::Derived));
rt = T_void;
Expand All @@ -4739,24 +4739,31 @@ static jl_returninfo_t get_specsig_function(Module *M, const std::string &name,
attributes = attributes.addAttribute(jl_LLVMContext, 1, Attribute::StructRet);
attributes = attributes.addAttribute(jl_LLVMContext, 1, Attribute::NoAlias);
attributes = attributes.addAttribute(jl_LLVMContext, 1, Attribute::NoCapture);
attributes = attributes.addDereferenceableAttr(jl_LLVMContext, 1, llvm_sizeof(jlrettype));
}
if (props.cc == jl_returninfo_t::Union) {
attributes = attributes.addAttribute(jl_LLVMContext, 1, Attribute::NoAlias);
attributes = attributes.addAttribute(jl_LLVMContext, 1, Attribute::NoCapture);
attributes = attributes.addDereferenceableAttr(jl_LLVMContext, 1, props.union_bytes);
}
for (size_t i = 0; i < jl_nparams(sig); i++) {
jl_value_t *jt = jl_tparam(sig, i);
bool isboxed;
Type *ty = julia_type_to_llvm(jt, &isboxed);
if (type_is_ghost(ty))
continue;
unsigned argno = fsig.size() + 1;
if (ty->isAggregateType()) { // aggregate types are passed by pointer
attributes = attributes.addAttribute(jl_LLVMContext, fsig.size() + 1, Attribute::NoCapture);
attributes = attributes.addAttribute(jl_LLVMContext, fsig.size() + 1, Attribute::ReadOnly);
attributes = attributes.addAttribute(jl_LLVMContext, argno, Attribute::NoCapture);
attributes = attributes.addAttribute(jl_LLVMContext, argno, Attribute::ReadOnly);
attributes = attributes.addDereferenceableAttr(jl_LLVMContext, argno, llvm_sizeof(jt));
ty = PointerType::get(ty, AddressSpace::Derived);
}
if (isboxed)
else if (isboxed) {
if (jl_is_leaf_type(jt))
attributes = attributes.addDereferenceableAttr(jl_LLVMContext, argno, jl_datatype_size(jt));
ty = PointerType::get(cast<PointerType>(ty)->getElementType(), AddressSpace::Tracked);
}
fsig.push_back(ty);
}
FunctionType *ftype = FunctionType::get(rt, fsig, false);
Expand Down Expand Up @@ -5263,15 +5270,12 @@ static std::unique_ptr<Module> emit_function(
theArg = ghostValue(argType);
}
else if (llvmArgType->isAggregateType()) {
Argument *Arg = &*AI++;
maybe_mark_argument_dereferenceable(Arg, argType);
Argument *Arg = &*AI; ++AI;
theArg = mark_julia_slot(Arg, argType, NULL, tbaa_const); // this argument is by-pointer
theArg.isimmutable = true;
}
else {
Argument *Arg = &*AI++;
if (isboxed)
maybe_mark_argument_dereferenceable(Arg, argType);
Argument *Arg = &*AI; ++AI;
theArg = mark_julia_type(ctx, Arg, isboxed, argType);
}
return theArg;
Expand Down Expand Up @@ -5697,8 +5701,8 @@ static std::unique_ptr<Module> emit_function(
if (retvalinfo.ispointer()) {
if (returninfo.cc == jl_returninfo_t::SRet) {
assert(jl_is_leaf_type(jlrettype));
emit_memcpy(ctx, sret, retvalinfo, jl_datatype_size(jlrettype),
julia_alignment(jlrettype, 0));
emit_memcpy(ctx, sret, retvalinfo, llvm_sizeof(jlrettype),
llvm_alignment(jlrettype, 0));
}
else { // must be jl_returninfo_t::Union
emit_unionmove(ctx, sret, retvalinfo, isboxed_union, false, NULL);
Expand Down
4 changes: 2 additions & 2 deletions src/intrinsics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va
return NULL;
}

unsigned alignment = julia_alignment(jt, 0);
unsigned alignment = llvm_alignment(jt, 0);
if (dest) {
MDNode *tbaa = x.tbaa;
// the memcpy intrinsic does not allow to specify different alias tags
Expand All @@ -353,7 +353,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va
// x.tbaa ∪ tbaa_stack = tbaa_root if x.tbaa != tbaa_stack
if (tbaa != tbaa_stack)
tbaa = NULL;
emit_memcpy(ctx, dest, p, jl_datatype_size(jt), alignment, volatile_store, tbaa);
emit_memcpy(ctx, dest, p, llvm_sizeof(jt), alignment, volatile_store, tbaa);
return NULL;
}
else {
Expand Down

0 comments on commit 05bc1b7

Please sign in to comment.