Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -601,6 +601,50 @@ const char* rocke_i_cmp_pred_error(const rocke_attr_map_t* attrs,
cap);
}

bool rocke_i_is_buffer_descriptor(const rocke_value_t* value)
{
return value && value->op && value->op->opcode == ROCKE_OP_TILE_BUFFER_RSRC;
}

const char* rocke_i_loop_carried_descriptor_message(int index, char* buf, size_t cap)
{
snprintf(buf,
cap,
"scf.for iter_arg %d is a buffer descriptor, which a loop cannot carry; build the "
"descriptor before the loop and advance the offset, or rebuild it inside the loop "
"body",
index);
return buf;
}

const char* rocke_i_scf_for_carried_descriptor_error(const rocke_op_t* op, char* buf, size_t cap)
{
int i;
for(i = 3; i < op->num_operands; ++i)
{
if(rocke_i_is_buffer_descriptor(op->operands[i]))
return rocke_i_loop_carried_descriptor_message(i - 3, buf, cap);
}
if(op->num_regions < 1 || !op->regions[0])
return NULL;
{
const rocke_region_t* body = op->regions[0];
int k;
for(k = 0; k < body->num_ops; ++k)
{
const rocke_op_t* body_op = body->ops[k];
if(body_op->opcode != ROCKE_OP_SCF_YIELD)
continue;
for(i = 0; i < body_op->num_operands; ++i)
{
if(rocke_i_is_buffer_descriptor(body_op->operands[i]))
return rocke_i_loop_carried_descriptor_message(i, buf, cap);
}
}
}
return NULL;
}

bool rocke_attr_get_bool(const rocke_attr_map_t* m, const char* key, bool dflt)
{
const rocke_attr_value_t* v = rocke_attr_get(m, key);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2219,6 +2219,18 @@ rocke_for_t rocke_b_scf_for_iter(rocke_ir_builder_t* b,
rocke_i_set_err(b, ROCKE_ERR_VALUE, "scf_for_iter: NULL iter_args");
return out;
}
for(i = 0; i < num_iter_args; ++i)
{
if(rocke_i_is_buffer_descriptor(iter_args[i].init))
{
char msg[ROCKE_ERR_MSG_CAP];
rocke_i_set_err(b,
ROCKE_ERR_VALUE,
"%s",
rocke_i_loop_carried_descriptor_message(i, msg, sizeof msg));
return out;
}
}

body = rocke_i_new_region(b, "body");
if(!body)
Expand Down Expand Up @@ -2378,9 +2390,28 @@ rocke_for_t rocke_b_scf_for_iter(rocke_ir_builder_t* b,
return out;
}

/* Python IRBuilder._current_region_is_loop_body: true while `b` is emitting
* into an scf.for body. The loop op is the last op of the region enclosing its
* body: it was emitted there before its body was entered, and every op since
* went into the body. */
static bool current_region_is_loop_body(const rocke_ir_builder_t* b)
{
const rocke_region_t* enclosing;
const rocke_op_t* loop;
if(b->region_depth < 2)
return false;
enclosing = b->region_stack[b->region_depth - 2];
if(enclosing->num_ops == 0)
return false;
loop = enclosing->ops[enclosing->num_ops - 1];
return loop->opcode == ROCKE_OP_SCF_FOR && loop->num_regions >= 1
&& loop->regions[0] == b->region_stack[b->region_depth - 1];
}

void rocke_b_scf_yield(rocke_ir_builder_t* b, rocke_value_t* const* values, int num_values)
{
rocke_attr_map_t attrs;
int i;
if(!rocke_i_live(b))
return;
if(num_values < 0)
Expand All @@ -2390,6 +2421,22 @@ void rocke_b_scf_yield(rocke_ir_builder_t* b, rocke_value_t* const* values, int
rocke_i_set_err(b, ROCKE_ERR_VALUE, "scf_yield: NULL values");
return;
}
/* Yielded values become the next iteration's iter_args in a loop body. */
if(current_region_is_loop_body(b))
{
for(i = 0; i < num_values; ++i)
{
if(rocke_i_is_buffer_descriptor(values[i]))
{
char msg[ROCKE_ERR_MSG_CAP];
rocke_i_set_err(b,
ROCKE_ERR_VALUE,
"%s",
rocke_i_loop_carried_descriptor_message(i, msg, sizeof msg));
return;
}
}
}
attrs = rocke_i_attrs(b);
rocke_attr_set_int(b, &attrs, "num", (int64_t)num_values);
rocke_i_op0(b, ROCKE_OP_SCF_YIELD, values, num_values, &attrs);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1438,6 +1438,13 @@ static rocke_status_t rocke_h_op_scf_for(rocke_h_lowerer_t* lw, const rocke_op_t
{
return lw->status;
}
{
char descriptor_error[ROCKE_ERR_MSG_CAP];
if(rocke_i_scf_for_carried_descriptor_error(op, descriptor_error, sizeof descriptor_error))
{
return rocke_h_fail(lw, ROCKE_ERR_VALUE, "%s", descriptor_error);
}
}
rocke_attr_get_int(&op->attrs, "num_iter_args", &num_iter);

iter_meta = rocke_attr_get(&op->attrs, "iter_args");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2069,10 +2069,20 @@ void rocke_ll_lower_unrolled_for(rocke_lower_t* L, const rocke_op_t* op)
}
}

/* Python _op_scf_for: pick unrolled vs normal lowering. */
/* Python _op_scf_for: refuse a loop-carried buffer descriptor, then pick
* unrolled vs normal lowering. */
static void _op_scf_for(rocke_lower_t* L, const rocke_op_t* op)
{
if(!rocke_ll_live(L) || op->num_operands < 3)
if(!rocke_ll_live(L))
{
return;
}
char descriptor_error[ROCKE_ERR_MSG_CAP];
if(rocke_i_scf_for_carried_descriptor_error(op, descriptor_error, sizeof descriptor_error))
{
rocke_ll_fail(L, ROCKE_ERR_VALUE, "%s", descriptor_error);
}
if(op->num_operands < 3)
{
return;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,13 @@ static void check_scf_for(verifier_t* v, const rocke_op_t* op)
}
}
}
{
char descriptor_error[ROCKE_ERR_MSG_CAP];
if(rocke_i_scf_for_carried_descriptor_error(op, descriptor_error, sizeof descriptor_error))
{
v_errf(v, op, "%s", descriptor_error);
}
}

/* body scope: snapshot + iv + iter-arg block values */
int saved = v->scope_count;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,26 @@ void rocke_i_check_bitcast_width(const char* verb,
const rocke_type_t* target,
const char* hint);

/* ------------------------------------------------- loop-carried descriptors */

/* Mirrors ir._is_buffer_descriptor: true when `value` is the result of a
* tile.buffer_rsrc op. */
bool rocke_i_is_buffer_descriptor(const rocke_value_t* value);

/* Mirrors ir._loop_carried_descriptor_message: writes into `buf` the refusal
* for a buffer descriptor as scf.for iter_arg `index` (its initial value or the
* value an iteration yields to it) and returns `buf`. A loop cannot carry a
* descriptor: the phi or HIP variable holding the carried value cannot hold the
* lowered descriptor (an opaque ptr addrspace(8) / rsrc_t). */
const char* rocke_i_loop_carried_descriptor_message(int index, char* buf, size_t cap);

/* Mirrors ir._scf_for_carried_descriptor_error: the refusal for the scf.for
* `op` when an iter_arg init (operands[3:]) or a value an scf.yield directly in
* its body yields is a buffer descriptor, written into `buf`; NULL otherwise.
* Accepts any operand / region counts. The builder, the verifier and both
* lowerings use it. */
const char* rocke_i_scf_for_carried_descriptor_error(const rocke_op_t* op, char* buf, size_t cap);

/* ------------------------------------------------------------- attr helpers */

/* Build a small attr map IN the arena and return it by value (the map's
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ All MFMAs take three literal immarg constants (`cbsz=0, abid=0, blgp=0`) at the
| `scf.yield` | `scf_yield(*values)` | records yielded values; consumed by enclosing for/if |
| `cf.return` | (auto-emitted) | `ret void` |

A loop cannot carry a buffer descriptor (`buffer_rsrc`), as an iter_arg's initial value or as a yielded value: it is `<4 x i32>` in the IR but an opaque `ptr addrspace(8)` (HIP `rsrc_t`) once lowered, which neither a phi nor a variable can hold. The builder, the verifier and every lowering refuse it; build the descriptor before the loop and advance the offset, or rebuild it inside the loop body.

`static_for` and `unroll` are Python-time helpers that emit no `scf.for`; they execute their body in Python and emit straight-line IR.

## Compile-Time Helpers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,59 @@ def _check_bitcast_width(verb: str, source: Type, target: Type, hint: str) -> No
)


def _is_buffer_descriptor(value: "Value") -> bool:
"""True when ``value`` is a buffer descriptor: the result of a
``tile.buffer_rsrc`` op.

A descriptor is a ``<4 x i32>`` in the IR but an opaque
``ptr addrspace(8)`` (HIP ``rsrc_t``) once lowered, so a value of that
vector type that does not come from ``tile.buffer_rsrc`` is not one.
"""
return value.op is not None and value.op.name == "tile.buffer_rsrc"


def _loop_carried_descriptor_message(index: int) -> str:
"""The refusal for a buffer descriptor as ``scf.for`` iter_arg ``index``,
either as its initial value or as the value a loop iteration yields to it.

A loop cannot carry a descriptor: the phi (LLVM) or the ``i32x4`` variable
(HIP) that holds a loop-carried value cannot hold the lowered descriptor,
and the compiler refuses the module. Carrying one is never needed either:
no op edits an existing descriptor, so a loop can only use the one built
before it (advancing the offset it passes to each access) or build a new
one in its body.
"""
return (
f"scf.for iter_arg {index} is a buffer descriptor, which a loop cannot "
"carry; build the descriptor before the loop and advance the offset, or "
"rebuild it inside the loop body"
)


def _scf_for_carried_descriptor_error(op: "Op") -> Optional[str]:
"""The refusal for the ``scf.for`` ``op`` when it carries a buffer
descriptor, or None when it does not.

It checks each iter_arg's initial value (the operands after lower, upper
and step) and each value an ``scf.yield`` directly in the loop body yields.
It only slices the operand and region lists, so it accepts any counts:
serialized IR reaches the verifier and the lowerings without passing the
builder. The verifier and every lowering of ``scf.for`` use it, and the C++
engine reports the same text (``rocke_i_scf_for_carried_descriptor_error``).
"""
for index, init in enumerate(op.operands[3:]):
if _is_buffer_descriptor(init):
return _loop_carried_descriptor_message(index)
for body in op.regions[:1]:
for body_op in body.ops:
if body_op.name != "scf.yield":
continue
for index, value in enumerate(body_op.operands):
if _is_buffer_descriptor(value):
return _loop_carried_descriptor_message(index)
return None


# ----------------------------- Values / Ops ------------------------------


Expand Down Expand Up @@ -5794,7 +5847,13 @@ def scf_for_iter(
elide_trailing_barrier: Phase 4 optimization (default: True)
- True: automatically elide trailing sync() in non-final iterations
- False: preserve all barriers (for manually optimized kernels)

No init may be a buffer descriptor (see
:func:`_loop_carried_descriptor_message`).
"""
for index, (_, init) in enumerate(iter_args):
if _is_buffer_descriptor(init):
raise ValueError(_loop_carried_descriptor_message(index))
body = Region("body")
iv = Value(name=f"%{iv_name}", type=lower.type)
iter_vars: List[Value] = []
Expand Down Expand Up @@ -5829,8 +5888,33 @@ def scf_for_iter(
return _ForBuilder(self, op, iv, body, iter_vars)

def scf_yield(self, *values: Value) -> None:
"""End the current region by yielding ``values`` to its owner.

In an ``scf.for`` body they become the next iteration's iter_args, so
none may be a buffer descriptor (see
:func:`_loop_carried_descriptor_message`).
"""
if self._current_region_is_loop_body():
for index, value in enumerate(values):
if _is_buffer_descriptor(value):
raise ValueError(_loop_carried_descriptor_message(index))
self._op("scf.yield", list(values), [], attrs={"num": len(values)})

def _current_region_is_loop_body(self) -> bool:
"""True while the builder is emitting into an ``scf.for`` body.

The loop op is the last op of the region that encloses its body: it was
emitted there before its body was entered, and every op since went into
the body.
"""
if len(self._region_stack) < 2:
return False
enclosing = self._region_stack[-2].ops
if not enclosing or enclosing[-1].name != "scf.for":
return False
loop_regions = enclosing[-1].regions
return bool(loop_regions) and loop_regions[0] is self._region_stack[-1]

def static_if(
self,
cond: bool,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
F64,
I32,
_int_add_wrap_error,
_scf_for_carried_descriptor_error,
KernelDef,
Op,
PtrType,
Expand Down Expand Up @@ -2749,6 +2750,9 @@ def _op_cf_return(self, op: Op) -> None:
# -------------------- control flow --------------------

def _op_scf_for(self, op: Op) -> None:
descriptor_error = _scf_for_carried_descriptor_error(op)
if descriptor_error is not None:
raise ValueError(descriptor_error)
num_iter = op.attrs.get("num_iter_args", 0)
lower = op.operands[0]
upper = op.operands[1]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
_FENCE_SCOPES,
_FLAG_STORE_ORDERINGS,
_int_add_wrap_error,
_scf_for_carried_descriptor_error,
BF16,
F32,
F16,
Expand Down Expand Up @@ -6950,8 +6951,12 @@ def _op_scf_for(self, op: Op) -> None:
"""Lower scf.for to LLVM IR.

Checks for 'unroll' hint and constant bounds to decide between
unrolled (straight-line) or normal (loop) lowering.
unrolled (straight-line) or normal (loop) lowering. Both refuse a
loop that carries a buffer descriptor.
"""
descriptor_error = _scf_for_carried_descriptor_error(op)
if descriptor_error is not None:
raise ValueError(descriptor_error)
unroll = op.attrs.get("unroll", False)
lower, upper, step = op.operands[:3]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
_int_add_wrap_error,
_int_cmp_pred_error,
_is_flag_ptr,
_scf_for_carried_descriptor_error,
KernelDef,
Op,
Region,
Expand Down Expand Up @@ -508,6 +509,9 @@ def _check_scf_for(self, op: Op, scope: Dict[str, Value]) -> None:
f"type {init.type.name}",
op,
)
descriptor_error = _scf_for_carried_descriptor_error(op)
if descriptor_error is not None:
self.err(descriptor_error, op)

# body scope: copy + iv + iter-arg block values
body_scope = dict(scope)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,22 @@ if(BUILD_TESTING)
rocke_buffer_rsrc_hip)
endif()

# A buffer descriptor refused as a loop-carried value by the C++ builder and
# verifier.
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/test_loop_carried_descriptor.cpp")
add_executable(rocke_loop_carried_descriptor core/test_loop_carried_descriptor.cpp)
target_link_libraries(rocke_loop_carried_descriptor PRIVATE rocke_core)
add_test(NAME rocke_loop_carried_descriptor COMMAND rocke_loop_carried_descriptor)
set_tests_properties(rocke_loop_carried_descriptor PROPERTIES LABELS "ckc;unit;host")
rocke_cxx_test_installed_name(_rocke_installed_name rocke_loop_carried_descriptor)
set_target_properties(rocke_loop_carried_descriptor
PROPERTIES OUTPUT_NAME "${_rocke_installed_name}")
install(TARGETS rocke_loop_carried_descriptor
RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR})
set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS
rocke_loop_carried_descriptor)
endif()

# Direct f32 tanh lowering and narrow-type rejection in the C++ engine.
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/test_tanh_lowering.cpp")
add_executable(rocke_tanh_lowering core/test_tanh_lowering.cpp)
Expand Down
Loading
Loading