Skip to content

perf(multimodal): replace scalar bicubic_resize with SIMD FIR CatmullRom and fuse phi3/phi4 normalize #929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
slin1237 wants to merge 2 commits into from
Closed

perf(multimodal): replace scalar bicubic_resize with SIMD FIR CatmullRom and fuse phi3/phi4 normalize #929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18cd4df
perf(multimodal): replace scalar bicubic_resize with SIMD FIR Catmull…
slin1237 Mar 26, 2026
835fc88
bench(multimodal): add phi3-vision preprocessing benchmarks
slin1237 Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion crates/multimodal/benches/image_preprocess.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ use image::{imageops::FilterType, DynamicImage, RgbImage};
use llm_multimodal::vision::{
image_processor::ImagePreProcessor,
preprocessor_config::PreProcessorConfig,
processors::{Llama4VisionProcessor, Qwen2VLProcessor, Qwen3VLProcessor},
processors::{
Llama4VisionProcessor, Phi3VisionProcessor, Qwen2VLProcessor, Qwen3VLProcessor,
},
transforms,
};

Expand Down Expand Up @@ -161,6 +163,27 @@ fn bench_llama4(c: &mut Criterion) {
group.finish();
}

fn bench_phi3_vision(c: &mut Criterion) {
let processor = Phi3VisionProcessor::new();
let config = PreProcessorConfig::default();

let sizes: &[(u32, u32)] = &[(224, 224), (336, 336), (640, 480), (1024, 768)];

let mut group = c.benchmark_group("phi3_vision_preprocess");
for &(w, h) in sizes {
let image = make_test_image(w, h);
let images = [image];
group.bench_with_input(
BenchmarkId::new("single", format!("{w}x{h}")),
&images,
|b, imgs| {
b.iter(|| processor.preprocess(imgs, &config).unwrap());
},
);
}
group.finish();
}

// ── Per-step profiling benchmarks ────────────────────────────────

fn bench_individual_steps(c: &mut Criterion) {
Expand Down Expand Up @@ -277,6 +300,7 @@ criterion_group!(
bench_qwen3_vl,
bench_qwen2_vl,
bench_llama4,
bench_phi3_vision,
bench_llama4_steps,
bench_individual_steps,
bench_fused_to_tensor_normalize,
Expand Down
31 changes: 18 additions & 13 deletions crates/multimodal/src/vision/processors/phi3_vision.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
//!
//! 1. **HD Transform**: Resize and pad image to multiples of 336
//! 2. **Normalize**: Apply CLIP normalization
//! 3. **Create Global Image**: Bicubic interpolate to 336x336
//! 3. **Create Global Image**: SIMD FIR CatmullRom resize to 336x336
//! 4. **Tile**: Reshape into (num_tiles, 3, 336, 336)
//! 5. **Concatenate**: [global_image, tiles...]
//! 6. **Pad**: Zero-pad to (num_crops+1, 3, 336, 336)
Expand Down Expand Up @@ -177,16 +177,20 @@ impl Phi3VisionProcessor {
new_image
}

/// Create global image by bicubic interpolation to 336x336.
///
/// Uses the shared `bicubic_resize` which matches PyTorch's
/// `torch.nn.functional.interpolate(mode='bicubic', align_corners=False)`.
/// Create global image by resizing the raw DynamicImage to TILE_SIZE x TILE_SIZE
/// using SIMD-accelerated FIR CatmullRom, then converting to a normalized tensor.
#[expect(
clippy::unused_self,
reason = "method logically belongs to the processor; keeps API consistent"
)]
fn create_global_image(&self, tensor: &Array3<f32>) -> Array3<f32> {
transforms::bicubic_resize(tensor, TILE_SIZE as usize, TILE_SIZE as usize)
fn create_global_image(
&self,
hd_image: &DynamicImage,
mean: &[f64; 3],
std: &[f64; 3],
) -> Array3<f32> {
let resized = transforms::resize(hd_image, TILE_SIZE, TILE_SIZE, FilterType::CatmullRom);
transforms::to_tensor_and_normalize(&resized, mean, std)
}

/// Reshape HD image into tiles.
Expand Down Expand Up @@ -243,12 +247,11 @@ impl Phi3VisionProcessor {
// 1. Convert to RGB
let image = DynamicImage::ImageRgb8(image.to_rgb8());

// 2. HD transform
// 2. HD transform (produces a DynamicImage)
let hd_image = self.hd_transform(&image);
let (hd_w, hd_h) = hd_image.dimensions();

// 3. To tensor [0, 1] and normalize
let mut tensor = transforms::to_tensor(&hd_image);
// Resolve normalization parameters
let mean = config
.image_mean
.as_ref()
Expand All @@ -259,10 +262,12 @@ impl Phi3VisionProcessor {
.as_ref()
.map(|v| [v[0], v[1], v[2]])
.unwrap_or(self.std);
transforms::normalize(&mut tensor, &mean, &std);

// 4. Create global image (336x336)
let global_image = self.create_global_image(&tensor);
// 3. Create global image: FIR CatmullRom resize on raw image, then fused to_tensor+normalize
let global_image = self.create_global_image(&hd_image, &mean, &std);

// 4. Fused to_tensor + normalize on HD image (single pass)
let tensor = transforms::to_tensor_and_normalize(&hd_image, &mean, &std);

// 5. Reshape HD image into tiles
let tiles = self.reshape_to_tiles(&tensor);
Expand Down
27 changes: 15 additions & 12 deletions crates/multimodal/src/vision/processors/phi4_vision.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,13 +302,17 @@ impl Phi4VisionProcessor {
DynamicImage::ImageRgb8(padded)
}

/// Create global image by bicubic interpolation to base resolution.
///
/// Uses the shared `bicubic_resize` which matches PyTorch's
/// `torch.nn.functional.interpolate(mode='bicubic', align_corners=False)`.
fn create_global_image(&self, tensor: &Array3<f32>) -> Array3<f32> {
let target = self.base_resolution as usize;
transforms::bicubic_resize(tensor, target, target)
/// Create global image by resizing the raw DynamicImage to base_resolution x base_resolution
/// using SIMD-accelerated FIR CatmullRom, then converting to a normalized tensor.
fn create_global_image(
&self,
hd_image: &DynamicImage,
mean: &[f64; 3],
std: &[f64; 3],
) -> Array3<f32> {
let target = self.base_resolution;
let resized = transforms::resize(hd_image, target, target, FilterType::CatmullRom);
transforms::to_tensor_and_normalize(&resized, mean, std)
}

/// Tile the HD image into crops of base_resolution x base_resolution.
Expand Down Expand Up @@ -385,12 +389,11 @@ impl Phi4VisionProcessor {
let hd_h = hd_image.height();
let hd_w = hd_image.width();

// Step 2: Convert to tensor and normalize
let mut hd_tensor = transforms::to_tensor(&hd_image);
transforms::normalize(&mut hd_tensor, &self.mean, &self.std);
// Step 2: Create global image: FIR CatmullRom resize on raw image, then fused to_tensor+normalize
let global_tensor = self.create_global_image(&hd_image, &self.mean, &self.std);

// Step 3: Create global image
let global_tensor = self.create_global_image(&hd_tensor);
// Step 3: Fused to_tensor + normalize on HD image (single pass)
let hd_tensor = transforms::to_tensor_and_normalize(&hd_image, &self.mean, &self.std);
Comment on lines +392 to +396

@coderabbitai coderabbitai Bot Mar 26, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Honor image_std-only overrides in this path.

Line 393 and Line 396 always use self.mean/self.std, but preprocess() only rebuilds a config-derived processor when dynamic_hd or image_mean is set. A PreProcessorConfig that overrides only image_std will still normalize with the default std here.

🔧 Minimal fix outside this hunk 
-        let processor = if config.dynamic_hd.is_some() || config.image_mean.is_some() {
+        let processor = if config.dynamic_hd.is_some()
+            || config.image_mean.is_some()
+            || config.image_std.is_some()
+        {
             Self::from_preprocessor_config(config)
         } else {
             self.clone()
         };
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@crates/multimodal/src/vision/processors/phi4_vision.rs` around lines 392 -
396, The code always passes self.mean/self.std to create_global_image and
transforms::to_tensor_and_normalize which ignores a PreProcessorConfig that
overrides only image_std; fix by computing effective mean/std that honor
optional overrides from PreProcessorConfig (e.g. use
config.image_mean.unwrap_or(self.mean) and config.image_std.unwrap_or(self.std)
or a helper like self.effective_mean_std()), then call
create_global_image(&hd_image, &effective_mean, &effective_std) and
transforms::to_tensor_and_normalize(&hd_image, &effective_mean, &effective_std);
also ensure preprocess() or the processor state exposes the PreProcessorConfig
used so image_std-only overrides are considered.


// Step 4: Tile HD image
let tiles = self.tile_image(&hd_tensor, h_crops, w_crops);
Expand Down
101 changes: 0 additions & 101 deletions crates/multimodal/src/vision/transforms.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,107 +369,6 @@ pub fn mean_to_rgb(mean: &[f64; 3]) -> Rgb<u8> {
])
}

/// Cubic interpolation weight function (Keys bicubic kernel with a=-0.5).
///
/// This matches PyTorch's bicubic interpolation used in
/// `torch.nn.functional.interpolate(mode='bicubic')`.
#[inline]
pub fn cubic_weight(x: f32) -> f32 {
let x = x.abs();
if x < 1.0 {
(1.5 * x - 2.5) * x * x + 1.0
} else if x < 2.0 {
((-0.5 * x + 2.5) * x - 4.0) * x + 2.0
} else {
0.0
}
}

/// Perform bicubic interpolation at a single point in a tensor.
///
/// Uses a 4x4 kernel with Keys bicubic weights (a=-0.5) to match PyTorch's
/// `torch.nn.functional.interpolate(mode='bicubic')`.
///
/// # Arguments
/// * `tensor` - Input tensor of shape [C, H, W]
/// * `c` - Channel index
/// * `src_y` - Source Y coordinate (can be fractional)
/// * `src_x` - Source X coordinate (can be fractional)
/// * `h` - Height of the tensor
/// * `w` - Width of the tensor
///
/// # Returns
/// The interpolated value at the specified position.
pub fn bicubic_interpolate(
tensor: &Array3<f32>,
c: usize,
src_y: f32,
src_x: f32,
h: usize,
w: usize,
) -> f32 {
let y_int = src_y.floor() as i32;
let x_int = src_x.floor() as i32;
let y_frac = src_y - y_int as f32;
let x_frac = src_x - x_int as f32;

let mut result = 0.0f32;

// Sample 4x4 neighborhood
for dy in -1..=2 {
let y_idx = (y_int + dy).clamp(0, h as i32 - 1) as usize;
let y_weight = cubic_weight(y_frac - dy as f32);

for dx in -1..=2 {
let x_idx = (x_int + dx).clamp(0, w as i32 - 1) as usize;
let x_weight = cubic_weight(x_frac - dx as f32);

result += tensor[[c, y_idx, x_idx]] * y_weight * x_weight;
}
}

result
}

/// Resize a tensor using bicubic interpolation.
///
/// This matches PyTorch's `torch.nn.functional.interpolate(mode='bicubic', align_corners=False)`.
///
/// # Arguments
/// * `tensor` - Input tensor of shape [C, H, W]
/// * `target_h` - Target height
/// * `target_w` - Target width
///
/// # Returns
/// Resized tensor of shape [C, target_h, target_w].
pub fn bicubic_resize(tensor: &Array3<f32>, target_h: usize, target_w: usize) -> Array3<f32> {
let (c, h, w) = (tensor.shape()[0], tensor.shape()[1], tensor.shape()[2]);

if h == target_h && w == target_w {
return tensor.clone();
}

let mut result = Array3::<f32>::zeros((c, target_h, target_w));

// PyTorch align_corners=False coordinate mapping
let scale_h = h as f32 / target_h as f32;
let scale_w = w as f32 / target_w as f32;

for ch in 0..c {
for y in 0..target_h {
for x in 0..target_w {
// PyTorch align_corners=False: src = (dst + 0.5) * scale - 0.5
let src_y = (y as f32 + 0.5) * scale_h - 0.5;
let src_x = (x as f32 + 0.5) * scale_w - 0.5;

result[[ch, y, x]] = bicubic_interpolate(tensor, ch, src_y, src_x, h, w);
}
}
}

result
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
Loading