Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions tools/mtmd/mtmd-helper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,29 @@ struct decode_embd_batch {
}
};

// Helper class to set non-causal attention via RAII
class scoped_non_causal_attention {
Comment thread
nikwen marked this conversation as resolved.
Resolved
public:
scoped_non_causal_attention(llama_context * context, bool enabled) : context_(context), enabled_(enabled) {
if (enabled_) {
// TODO @ngxson : need to make sure only one image is processed at a time, and n_ubatch must be enough to hold the image
llama_set_causal_attn(context_, false);
}
}
~scoped_non_causal_attention() {
if (enabled_) {
llama_set_causal_attn(context_, true);
}
}

scoped_non_causal_attention(const scoped_non_causal_attention &) = delete;
scoped_non_causal_attention & operator=(const scoped_non_causal_attention &) = delete;

private:
llama_context * context_;
bool enabled_;
};

// Helper function for decoding an image whose embeddings have already been calculated
int32_t mtmd_helper_decode_image_chunk(
mtmd_context * ctx,
Expand Down Expand Up @@ -288,10 +311,7 @@ int32_t mtmd_helper_decode_image_chunk(
}

const bool use_non_causal = mtmd_decode_use_non_causal(ctx, chunk);
if (use_non_causal) {
llama_set_causal_attn(lctx, false);
// TODO @ngxson : need to make sure only one image is processed at a time, and n_ubatch must be enough to hold the image
}
const scoped_non_causal_attention scoped_non_causal_attn(lctx, use_non_causal);

while (i_batch < n_img_batches) { // split into batches
int pos_offset = i_batch*n_batch;
Expand All @@ -304,19 +324,13 @@ int32_t mtmd_helper_decode_image_chunk(
int32_t ret = llama_decode(lctx, batch_embd_view);
if (ret != 0) {
LOG_ERR("failed to decode %s\n", name);
if (use_non_causal) {
llama_set_causal_attn(lctx, true);
}
return ret;
}

if (callback != nullptr) {
ret = callback(batch_embd_view, user_data);
if (ret != 0) {
LOG_ERR("post-decode callback failed\n");
if (use_non_causal) {
llama_set_causal_attn(lctx, true);
}
return ret;
}
}
Expand All @@ -329,9 +343,6 @@ int32_t mtmd_helper_decode_image_chunk(
n_past += mtmd_input_chunk_get_n_pos(chunk);
*new_n_past = n_past;

if (use_non_causal) {
llama_set_causal_attn(lctx, true);
}
return 0;
}

Expand Down