Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions csrc/cpu/cpu_attn.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,9 @@ torch::Tensor get_scheduler_metadata(
input.causal = causal;
input.isa = isa;
input.enable_kv_split = enable_kv_split;
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
input.dynamic_causal = dynamic_causal.has_value()
? dynamic_causal->data_ptr<int32_t>()
: nullptr;

const int64_t kv_cache_idx =
static_cast<int64_t>(parse_fp8_kv_dtype(kv_cache_dtype));
Expand Down Expand Up @@ -256,8 +257,9 @@ void cpu_attention_with_kv_cache(
input.s_aux = nullptr;
input.s_aux_is_bf16 = false;
}
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
input.dynamic_causal = dynamic_causal.has_value()
? dynamic_causal->data_ptr<int32_t>()
: nullptr;
input.scale = scale;
input.causal = causal;
input.sliding_window_size = sliding_window;
Expand Down
13 changes: 7 additions & 6 deletions csrc/cpu/cpu_attn_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ class AttentionScheduler {
int32_t max_num_q_per_iter; // max Q head num can be hold in registers
int32_t kv_block_alignment; // context length alignment requirement
bool enable_kv_split;
bool* dynamic_causal;
int32_t* dynamic_causal;
};

static constexpr int32_t MaxQTileIterNum = 128;
Expand Down Expand Up @@ -450,7 +450,7 @@ class AttentionScheduler {
const int32_t q_token_num =
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
const bool req_causal =
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
is_dynamic_causal ? input.dynamic_causal[req_id] != 0 : causal;
const int32_t q_start_pos = seq_len - q_token_num;
const int32_t kv_start_pos = 0;
const int32_t kv_end_pos = seq_len;
Expand Down Expand Up @@ -492,7 +492,7 @@ class AttentionScheduler {
const int32_t q_token_num =
input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
const bool req_causal =
is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
is_dynamic_causal ? input.dynamic_causal[req_id] != 0 : causal;
const int32_t q_start_pos = seq_len - q_token_num;
const int32_t kv_start_pos = 0;
const int32_t kv_end_pos = seq_len;
Expand Down Expand Up @@ -884,7 +884,7 @@ struct AttentionInput {
// provided as fp32 and executed in full float precision.
const void* s_aux;
bool s_aux_is_bf16;
bool* dynamic_causal;
int32_t* dynamic_causal;
float scale;
bool causal;
int32_t sliding_window_size;
Expand Down Expand Up @@ -1571,7 +1571,7 @@ class AttentionMainLoop {
const float* alibi_slopes = input->alibi_slopes;
const void* s_aux = input->s_aux;
const bool s_aux_is_bf16 = input->s_aux_is_bf16;
const bool* dynamic_causal = input->dynamic_causal;
const int32_t* dynamic_causal = input->dynamic_causal;
const bool is_dynamic_causal = dynamic_causal != nullptr;

const bool causal = input->causal;
Expand Down Expand Up @@ -1658,7 +1658,8 @@ class AttentionMainLoop {

const int32_t current_group_idx = current_workitem_group->req_id;
const int32_t current_group_causal =
is_dynamic_causal ? dynamic_causal[current_group_idx] : causal;
is_dynamic_causal ? dynamic_causal[current_group_idx] != 0
: causal;
auto [sliding_window_left, sliding_window_right] =
AttentionScheduler::calcu_sliding_window_size(
sliding_window_size, current_group_causal);
Expand Down
2 changes: 1 addition & 1 deletion tests/kernels/attention/test_cpu_attn.py
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ def varlen_with_paged_kv(
scale = head_size**-0.5
token_num = sum(query_lens)
dynamic_causal_tensor = (
torch.tensor(dynamic_causal, dtype=torch.bool)
torch.tensor(dynamic_causal, dtype=torch.int32)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this note as a heads up.

Since tying the test to the model's buffer felt out of scope here, I only updated this dtype.

However, note that the kernel test still builds its own mask instead of reading DiffusionGemma's _causal_buf. As a consequence, a future dtype change in the model would not fail this test. It would fail at the kernel instead, because data_ptr<int32_t>() checks the dtype.

if dynamic_causal is not None
else None
)
Expand Down
Loading