Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 28 additions & 5 deletions src/models/qwen_vl_vision.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -249,32 +249,55 @@ std::vector<float> QwenVisionPipeline::Run(const float* pixel_data, const std::v
// Matches HuggingFace transformers implementation:
// https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L367
std::vector<int64_t> QwenVisionPipeline::CalculateWindowIndex(int64_t grid_t, int64_t grid_h, int64_t grid_w) {
// Validate grid dimensions are positive and within reasonable bounds
if (grid_t <= 0 || grid_h <= 0 || grid_w <= 0) {
throw std::runtime_error("CalculateWindowIndex: grid dimensions must be positive");
}
Comment thread
apsonawane marked this conversation as resolved.
Outdated
Comment thread
apsonawane marked this conversation as resolved.
Outdated

// Calculate LLM grid dimensions after spatial merging
int64_t llm_grid_h = grid_h / spatial_merge_size_;
int64_t llm_grid_w = grid_w / spatial_merge_size_;

// Calculate window size at the merged resolution
int64_t vit_merger_window_size = window_size_ / spatial_merge_size_ / patch_size_;
if (vit_merger_window_size <= 0) {
throw std::runtime_error("CalculateWindowIndex: vit_merger_window_size must be positive (check window_size, spatial_merge_size, patch_size config)");
}
Comment thread
apsonawane marked this conversation as resolved.
Outdated

// Validate merged grid dimensions before computing padding and allocation
constexpr int64_t kMaxElements = static_cast<int64_t>(1) << 30; // ~1 billion elements, ~8GB
if (llm_grid_h > kMaxElements || llm_grid_w > kMaxElements || grid_t > kMaxElements) {
throw std::runtime_error("CalculateWindowIndex: grid dimensions are too large");
}

// Calculate padding needed to fit into windows
int64_t pad_h = (vit_merger_window_size - (llm_grid_h % vit_merger_window_size)) % vit_merger_window_size;
int64_t pad_w = (vit_merger_window_size - (llm_grid_w % vit_merger_window_size)) % vit_merger_window_size;

int64_t num_windows_h = (llm_grid_h + pad_h) / vit_merger_window_size;
int64_t num_windows_w = (llm_grid_w + pad_w) / vit_merger_window_size;
int64_t padded_h = llm_grid_h + pad_h;
int64_t padded_w = llm_grid_w + pad_w;

// Use division-based overflow check: grid_t * padded_h * padded_w <= kMaxElements
if (padded_h > 0 && padded_w > 0 && grid_t > kMaxElements / padded_h / padded_w) {
throw std::runtime_error("CalculateWindowIndex: total grid size exceeds maximum allowed");
}
int64_t alloc_size = grid_t * padded_h * padded_w;

int64_t num_windows_h = padded_h / vit_merger_window_size;
int64_t num_windows_w = padded_w / vit_merger_window_size;

std::vector<int64_t> window_index;
window_index.reserve(grid_t * llm_grid_h * llm_grid_w);

// Create initial index grid
std::vector<int64_t> index(grid_t * (llm_grid_h + pad_h) * (llm_grid_w + pad_w), -100);
std::vector<int64_t> index(alloc_size, -100);

// Fill non-padded positions with sequential indices
for (int64_t t = 0; t < grid_t; ++t) {
for (int64_t h = 0; h < llm_grid_h; ++h) {
for (int64_t w = 0; w < llm_grid_w; ++w) {
int64_t idx = t * llm_grid_h * llm_grid_w + h * llm_grid_w + w;
int64_t padded_idx = t * (llm_grid_h + pad_h) * (llm_grid_w + pad_w) + h * (llm_grid_w + pad_w) + w;
int64_t padded_idx = t * padded_h * padded_w + h * padded_w + w;
index[padded_idx] = idx;
}
}
Expand All @@ -290,7 +313,7 @@ std::vector<int64_t> QwenVisionPipeline::CalculateWindowIndex(int64_t grid_t, in
for (int64_t pw = 0; pw < vit_merger_window_size; ++pw) {
int64_t h = wh * vit_merger_window_size + ph;
int64_t w = ww * vit_merger_window_size + pw;
int64_t padded_idx = t * (llm_grid_h + pad_h) * (llm_grid_w + pad_w) + h * (llm_grid_w + pad_w) + w;
int64_t padded_idx = t * padded_h * padded_w + h * padded_w + w;

// Only add non-padded indices
if (index[padded_idx] != -100) {
Expand Down
Loading