diff --git a/crates/skippy-bench/src/cli.rs b/crates/skippy-bench/src/cli.rs index db456ffa85..a1ec2c126f 100644 --- a/crates/skippy-bench/src/cli.rs +++ b/crates/skippy-bench/src/cli.rs @@ -4,6 +4,19 @@ use clap::{Parser, Subcommand, ValueEnum}; pub const DEFAULT_LOCAL_MODEL_ID: &str = "jc-builds/SmolLM2-135M-Instruct-Q4_K_M-GGUF:Q4_K_M"; pub const DEFAULT_RUN_MAX_NEW_TOKENS: usize = 1; +pub const MAX_VERIFY_WINDOW_WIDTH: usize = 16; + +fn parse_verify_window_width(value: &str) -> Result { + let width = value + .parse::() + .map_err(|error| format!("invalid verification width: {error}"))?; + if !(1..=MAX_VERIFY_WINDOW_WIDTH).contains(&width) { + return Err(format!( + "verification width must be between 1 and {MAX_VERIFY_WINDOW_WIDTH}" + )); + } + Ok(width) +} #[derive(Parser)] #[command(about = "Llama stage benchmark launcher")] @@ -68,6 +81,14 @@ impl EvalId { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[clap(rename_all = "kebab-case")] +pub enum FlashAttentionArg { + Auto, + Disabled, + Enabled, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] #[clap(rename_all = "kebab-case")] pub enum EvalPack { @@ -236,6 +257,21 @@ pub struct VerifyWindowLocalArgs { pub iterations: usize, #[arg(long, default_value_t = 8)] pub warmup: usize, + /// Verification widths checked for canonical parity (maximum 16). + #[arg( + long, + value_delimiter = ',', + value_parser = parse_verify_window_width, + default_value = "2" + )] + pub verify_widths: Vec, + /// Independent verification width used for timing samples (maximum 16). + #[arg(long, value_parser = parse_verify_window_width, default_value_t = 2)] + pub sample_width: usize, + #[arg(long, default_value_t = 1)] + pub continuation_steps: usize, + #[arg(long = "flash-attn", value_enum, default_value = "auto")] + pub flash_attn: FlashAttentionArg, #[arg( long, default_value = "Write a Rust function that parses a list of integers and returns the median." @@ -584,7 +620,7 @@ mod tests { use clap::Parser; - use super::{Cli, CommandKind, EvalCommandKind, FocusedRuntimeScenario}; + use super::{Cli, CommandKind, EvalCommandKind, FlashAttentionArg, FocusedRuntimeScenario}; #[test] fn parses_eval_run_metrics_finalize_only() { @@ -687,6 +723,10 @@ mod tests { assert_eq!(args.iterations, 3); assert_eq!(args.warmup, 1); assert_eq!(args.n_gpu_layers, -1); + assert_eq!(args.verify_widths, vec![2]); + assert_eq!(args.sample_width, 2); + assert_eq!(args.continuation_steps, 1); + assert_eq!(args.flash_attn, FlashAttentionArg::Auto); } #[test] @@ -707,4 +747,106 @@ mod tests { assert_eq!(args.split_layer, Some(24)); } + + #[test] + fn parses_verify_window_local_widths() { + let cli = Cli::try_parse_from([ + "skippy-bench", + "verify-window-local", + "--model-path", + "/tmp/model.gguf", + "--verify-widths", + "1,2,4,9", + ]) + .unwrap(); + + let CommandKind::VerifyWindowLocal(args) = cli.command else { + panic!("expected verify-window-local subcommand"); + }; + + assert_eq!(args.verify_widths, vec![1, 2, 4, 9]); + } + + #[test] + fn parses_verify_window_local_sample_width() { + let cli = Cli::try_parse_from([ + "skippy-bench", + "verify-window-local", + "--model-path", + "/tmp/model.gguf", + "--verify-widths", + "1,2,4,9", + "--sample-width", + "9", + ]) + .unwrap(); + + let CommandKind::VerifyWindowLocal(args) = cli.command else { + panic!("expected verify-window-local subcommand"); + }; + + assert_eq!(args.verify_widths, vec![1, 2, 4, 9]); + assert_eq!(args.sample_width, 9); + } + + #[test] + fn rejects_verify_window_local_widths_above_kernel_ceiling() { + let widths = Cli::try_parse_from([ + "skippy-bench", + "verify-window-local", + "--model-path", + "/tmp/model.gguf", + "--verify-widths", + "1,17", + ]); + assert!(widths.is_err()); + + let sample = Cli::try_parse_from([ + "skippy-bench", + "verify-window-local", + "--model-path", + "/tmp/model.gguf", + "--sample-width", + "17", + ]); + assert!(sample.is_err()); + } + + #[test] + fn parses_verify_window_local_continuation_steps() { + let cli = Cli::try_parse_from([ + "skippy-bench", + "verify-window-local", + "--model-path", + "/tmp/model.gguf", + "--continuation-steps", + "256", + ]) + .unwrap(); + + let CommandKind::VerifyWindowLocal(args) = cli.command else { + panic!("expected verify-window-local subcommand"); + }; + + assert_eq!(args.continuation_steps, 256); + } + + #[test] + fn parses_verify_window_local_flash_attention() { + let cli = Cli::try_parse_from([ + "skippy-bench", + "verify-window-local", + "--model-path", + "/tmp/model.gguf", + "--flash-attn", + "disabled", + ]) + .unwrap(); + + let CommandKind::VerifyWindowLocal(args) = cli.command else { + panic!("expected verify-window-local subcommand"); + }; + + assert_eq!(args.flash_attn, FlashAttentionArg::Disabled); + } } diff --git a/crates/skippy-bench/src/verify_window_local.rs b/crates/skippy-bench/src/verify_window_local.rs index f47a195d76..8cfc428574 100644 --- a/crates/skippy-bench/src/verify_window_local.rs +++ b/crates/skippy-bench/src/verify_window_local.rs @@ -7,11 +7,11 @@ use std::{ use anyhow::{Context, Result, bail}; use serde::Serialize; use skippy_runtime::{ - FlashAttentionType, RuntimeConfig, RuntimeLoadMode, SamplingConfig, StageModel, StageSession, - parse_cache_type, + FlashAttentionType, GenerationSignalWindow, RuntimeConfig, RuntimeLoadMode, SamplingConfig, + StageModel, StageSession, TokenSignal, parse_cache_type, }; -use crate::cli::VerifyWindowLocalArgs; +use crate::cli::{FlashAttentionArg, MAX_VERIFY_WINDOW_WIDTH, VerifyWindowLocalArgs}; #[derive(Debug, Serialize)] struct TimingStats { @@ -46,13 +46,18 @@ struct VerifyWindowLocalReport { n_ubatch: Option, cache_type_k: String, cache_type_v: String, + flash_attn: &'static str, prompt_token_count: usize, verify_tokens: Vec, + verify_widths: Vec, + continuation_steps: usize, + parity_checks: Vec, warmup: usize, iterations: usize, - batched_width2: TimingStats, - serial_two_decode_mtp_n1: TimingStats, - split_inprocess_width2: Option, + sample_width: usize, + batched: TimingStats, + serial: TimingStats, + split_inprocess: Option, batched_avg_vs_serial_avg: f64, batched_token_per_sec: f64, serial_token_per_sec: f64, @@ -60,6 +65,30 @@ struct VerifyWindowLocalReport { first_serial_prediction: Vec, } +#[derive(Debug, Serialize)] +struct VerifyParityCheck { + width: usize, + matched: bool, + first_mismatch_position: Option, + target_stream_matched: bool, + continuation_matched: bool, + continuation_steps: usize, + native_position_matched: bool, + full_state_matched: bool, + serial_full_state_bytes: usize, + batched_full_state_bytes: usize, + token_signal_matched: bool, + signal_window_matched: bool, + serial_top_token: i32, + serial_second_token: i32, + serial_top2_margin: f32, + batched_top_token: i32, + batched_second_token: i32, + batched_top2_margin: f32, + batched_us: u128, + serial_us: u128, +} + #[derive(Debug, Serialize)] struct SplitInprocessReport { split_layer: u32, @@ -105,6 +134,11 @@ pub fn verify_window_local(args: VerifyWindowLocalArgs) -> Result<()> { None => None, }; let report = build_report(args, full, split)?; + let parity_failure = report + .parity_checks + .iter() + .find(|check| !check.matched) + .map(format_parity_failure); let encoded = serde_json::to_vec_pretty(&report)?; if let Some(path) = output { @@ -112,6 +146,9 @@ pub fn verify_window_local(args: VerifyWindowLocalArgs) -> Result<()> { .with_context(|| format!("failed to write {}", path.display()))?; } println!("{}", String::from_utf8(encoded)?); + if let Some(failure) = parity_failure { + bail!("{failure}"); + } Ok(()) } @@ -122,6 +159,10 @@ fn validate_args(args: &VerifyWindowLocalArgs) -> Result<()> { if args.iterations == 0 { bail!("iterations must be greater than zero"); } + validate_verify_widths(&args.verify_widths, args.sample_width)?; + if args.continuation_steps == 0 { + bail!("continuation_steps must be positive"); + } if let Some(split_layer) = args.split_layer && (split_layer == 0 || split_layer >= args.layer_end) { @@ -130,6 +171,38 @@ fn validate_args(args: &VerifyWindowLocalArgs) -> Result<()> { Ok(()) } +fn validate_verify_widths(verify_widths: &[usize], sample_width: usize) -> Result<()> { + if verify_widths.is_empty() || verify_widths.contains(&0) { + bail!("verify_widths must contain at least one positive width"); + } + if verify_widths + .iter() + .any(|&width| width > MAX_VERIFY_WINDOW_WIDTH) + { + bail!( + "verify_widths must not exceed the batch-invariant Metal ceiling of {MAX_VERIFY_WINDOW_WIDTH}" + ); + } + if verify_widths + .iter() + .enumerate() + .any(|(index, width)| verify_widths[..index].contains(width)) + { + bail!("verify_widths must not contain duplicates"); + } + if sample_width == 0 || sample_width > MAX_VERIFY_WINDOW_WIDTH { + bail!( + "sample_width must be between 1 and the batch-invariant Metal ceiling of {MAX_VERIFY_WINDOW_WIDTH}" + ); + } + if !verify_widths.contains(&sample_width) { + bail!( + "sample_width must also appear in verify_widths so timed execution is parity-checked" + ); + } + Ok(()) +} + fn full_runtime_config(args: &VerifyWindowLocalArgs) -> Result { Ok(RuntimeConfig { stage_index: 0, @@ -147,7 +220,7 @@ fn full_runtime_config(args: &VerifyWindowLocalArgs) -> Result { selected_backend_device: None, cache_type_k: parse_cache_type(&args.cache_type_k)?, cache_type_v: parse_cache_type(&args.cache_type_v)?, - flash_attn_type: FlashAttentionType::Auto, + flash_attn_type: runtime_flash_attn(args.flash_attn), load_mode: RuntimeLoadMode::RuntimeSlice, projector_path: None, include_embeddings: true, @@ -159,6 +232,7 @@ fn full_runtime_config(args: &VerifyWindowLocalArgs) -> Result { struct FullModelSamples { tokens: Vec, verify_tokens: Vec, + parity_checks: Vec, samples: SampleSet, } @@ -178,13 +252,23 @@ fn run_full_model_samples(args: &VerifyWindowLocalArgs) -> Result Result, +} + +enum ParityExecution { + Serial, + Batched, +} + +struct ParitySide { + prediction: Vec, + continuation: Vec, + position: u64, + full_state: Vec, + signal: TokenSignal, + signal_window: GenerationSignalWindow, + elapsed_us: u128, +} + +struct ParityRun<'a> { + base_token_count: u64, + layer_end: i32, + verify_tokens: &'a [i32], + continuation_input: i32, + width: usize, + continuation_steps: usize, +} + +impl VerifyTargetPlan { + fn verify_tokens_for_width(&self, width: usize) -> Result> { + if width == 0 { + bail!("verify width must be greater than zero"); + } + let required_targets = width.saturating_sub(1); + if self.targets.len() < required_targets { + bail!( + "target stream has {} token(s), but width {width} requires {required_targets} target token(s)", + self.targets.len(), + ); + } + let mut tokens = Vec::with_capacity(width); + tokens.push(self.current); + tokens.extend_from_slice(&self.targets[..width.saturating_sub(1)]); + Ok(tokens) + } + + fn expected_for_width(&self, width: usize) -> Result<&[i32]> { + if self.targets.len() < width { + bail!( + "target stream has {} token(s), but width {width} requires {width}", + self.targets.len() + ); + } + Ok(&self.targets[..width]) + } +} + +fn choose_target_plan( session: &mut StageSession, base_token_count: u64, prompt_tokens: &[i32], prompt: &str, config: &RuntimeConfig, -) -> Result> { + target_token_count: usize, +) -> Result { + // Kernel parity needs an always-accepted target stream. Native MTP proposal + // quality and acceptance remain covered by the dedicated MTP benchmarks. session .trim_session(base_token_count) - .context("failed to trim session before choosing verify tokens")?; + .context("failed to trim session before choosing verify target stream")?; let current = *prompt_tokens .first() .context("prompt produced no token for verify-token seed")?; - let (_after_current, native_mtp, _frame) = session - .decode_step_frame_sampled_mtp(current, Some(&SamplingConfig::default()), None, 0, 1) - .with_context(|| { - format!( - "failed to get native MTP draft from {} after prompt {:?}", - model_description(config), - prompt - ) - })?; - let Some(draft) = native_mtp else { - bail!("model did not produce a native MTP n=1 draft token"); - }; + let mut input = current; + let mut targets = Vec::with_capacity(target_token_count); + for _ in 0..target_token_count { + let (predicted, _native_mtp, _frame) = session + .decode_step_frame_sampled_mtp(input, Some(&SamplingConfig::default()), None, 0, 1) + .with_context(|| { + format!( + "failed to extend greedy target stream from {} after prompt {:?}", + model_description(config), + prompt + ) + })?; + if predicted < 0 { + bail!("target stream decode did not return a token"); + } + targets.push(predicted); + input = predicted; + } session .trim_session(base_token_count) - .context("failed to trim session after choosing verify tokens")?; - let draft_token = draft - .token_ids - .first() + .context("failed to trim session after choosing verify target stream")?; + Ok(VerifyTargetPlan { current, targets }) +} + +fn run_parity_checks( + session: &mut StageSession, + base_token_count: u64, + layer_end: u32, + target_plan: &VerifyTargetPlan, + widths: &[usize], + continuation_steps: usize, +) -> Result> { + let mut checks = Vec::with_capacity(widths.len()); + for &width in widths { + checks.push(run_parity_check( + session, + base_token_count, + layer_end, + target_plan, + width, + continuation_steps, + )?); + } + Ok(checks) +} + +fn run_parity_check( + session: &mut StageSession, + base_token_count: u64, + layer_end: u32, + target_plan: &VerifyTargetPlan, + width: usize, + continuation_steps: usize, +) -> Result { + let verify_tokens = target_plan.verify_tokens_for_width(width)?; + let expected = target_plan.expected_for_width(width)?; + let continuation_expected = expected_continuation(target_plan, width, continuation_steps)?; + let continuation_input = *target_plan + .targets + .get(width.saturating_sub(1)) + .with_context(|| format!("missing continuation input for width {width}"))?; + let layer_end = i32::try_from(layer_end).context("layer_end exceeds i32")?; + let run = ParityRun { + base_token_count, + layer_end, + verify_tokens: &verify_tokens, + continuation_input, + width, + continuation_steps, + }; + let serial = collect_parity_side(session, &run, ParityExecution::Serial)?; + let batched = collect_parity_side(session, &run, ParityExecution::Batched)?; + build_parity_check( + width, + continuation_steps, + expected, + continuation_expected, + &serial, + &batched, + ) +} + +fn collect_parity_side( + session: &mut StageSession, + run: &ParityRun<'_>, + execution: ParityExecution, +) -> Result { + let label = match execution { + ParityExecution::Serial => "serial", + ParityExecution::Batched => "batched", + }; + session + .trim_session(run.base_token_count) + .with_context(|| format!("failed to trim session before {label} parity check"))?; + let start = Instant::now(); + let prediction = match execution { + ParityExecution::Serial => serial_decode_expected(session, run.verify_tokens)?, + ParityExecution::Batched => { + session + .verify_tokens_frame_sampled( + run.verify_tokens, + Some(&SamplingConfig::default()), + None, + 0, + ) + .with_context(|| format!("batched width-{} VerifyWindow failed", run.width))? + .0 + } + }; + let elapsed_us = start.elapsed().as_micros(); + let position = session.token_count(); + let continuation = decode_continuation( + session, + run.continuation_input, + run.width, + run.continuation_steps, + )?; + let full_state = session + .export_full_state(0, run.layer_end) + .with_context(|| format!("failed to export {label} full state"))?; + let signal = session.last_token_signal()?; + let window_tokens = + u32::try_from(run.width.saturating_add(1)).context("signal window width exceeds u32")?; + let signal_window = session.signal_window(window_tokens)?; + Ok(ParitySide { + prediction, + continuation, + position, + full_state, + signal, + signal_window, + elapsed_us, + }) +} + +fn build_parity_check( + width: usize, + continuation_steps: usize, + expected: &[i32], + continuation_expected: &[i32], + serial: &ParitySide, + batched: &ParitySide, +) -> Result { + let batched_prefix = prediction_prefix(&batched.prediction, width)?; + let serial_prefix = prediction_prefix(&serial.prediction, width)?; + let target_stream_matched = batched_prefix == expected && serial_prefix == expected; + let continuation_matched = batched.continuation == continuation_expected + && serial.continuation == continuation_expected; + let native_position_matched = batched.position == serial.position; + let full_state_matched = batched.full_state == serial.full_state; + let token_signal_matched = batched.signal == serial.signal; + let signal_window_matched = batched.signal_window == serial.signal_window; + let first_mismatch_position = first_mismatch_position(batched_prefix, expected) + .or_else(|| first_mismatch_position(serial_prefix, expected)); + // Signal windows summarize decode-call history, so one batched call and N serial + // calls have different window shapes by construction. Keep this diagnostic + // visible, but gate parity on canonical model state and token signals. + let matched = target_stream_matched + && continuation_matched + && native_position_matched + && full_state_matched + && token_signal_matched; + Ok(VerifyParityCheck { + width, + matched, + first_mismatch_position, + target_stream_matched, + continuation_matched, + continuation_steps, + native_position_matched, + full_state_matched, + serial_full_state_bytes: serial.full_state.len(), + batched_full_state_bytes: batched.full_state.len(), + token_signal_matched, + signal_window_matched, + serial_top_token: serial.signal.top_token, + serial_second_token: serial.signal.second_token, + serial_top2_margin: serial.signal.margin, + batched_top_token: batched.signal.top_token, + batched_second_token: batched.signal.second_token, + batched_top2_margin: batched.signal.margin, + batched_us: batched.elapsed_us, + serial_us: serial.elapsed_us, + }) +} + +fn format_parity_failure(check: &VerifyParityCheck) -> String { + format!( + "verify parity failed at width {}: first_mismatch_position={:?} target_stream_matched={} continuation_matched={} native_position_matched={} full_state_matched={} serial_full_state_bytes={} batched_full_state_bytes={} token_signal_matched={} signal_window_matched={} serial_margin={} batched_margin={} serial_top={} batched_top={} batched_us={} serial_us={}", + check.width, + check.first_mismatch_position, + check.target_stream_matched, + check.continuation_matched, + check.native_position_matched, + check.full_state_matched, + check.serial_full_state_bytes, + check.batched_full_state_bytes, + check.token_signal_matched, + check.signal_window_matched, + check.serial_top2_margin, + check.batched_top2_margin, + check.serial_top_token, + check.batched_top_token, + check.batched_us, + check.serial_us + ) +} + +fn serial_decode_expected(session: &mut StageSession, verify_tokens: &[i32]) -> Result> { + let mut predicted_tokens = Vec::with_capacity(verify_tokens.len()); + for token_id in verify_tokens { + let (predicted, _native_mtp, _frame) = session + .decode_step_frame_sampled_mtp(*token_id, Some(&SamplingConfig::default()), None, 0, 1) + .context("serial target-stream decode failed")?; + if predicted < 0 { + bail!("serial target-stream decode returned no token for input {token_id}"); + } + predicted_tokens.push(predicted); + } + Ok(predicted_tokens) +} + +fn expected_continuation( + target_plan: &VerifyTargetPlan, + width: usize, + continuation_steps: usize, +) -> Result<&[i32]> { + let end = width + .checked_add(continuation_steps) + .with_context(|| format!("continuation index overflow at width {width}"))?; + target_plan.targets.get(width..end).with_context(|| { + format!( + "missing continuation targets for width {width}; target plan has {} token(s)", + target_plan.targets.len() + ) + }) +} + +fn decode_continuation( + session: &mut StageSession, + mut input: i32, + width: usize, + continuation_steps: usize, +) -> Result> { + let mut predicted_tokens = Vec::with_capacity(continuation_steps); + for step in 0..continuation_steps { + let (predicted, _native_mtp, _frame) = session + .decode_step_frame_sampled_mtp(input, Some(&SamplingConfig::default()), None, 0, 1) + .with_context(|| format!("continuation decode failed at width {width}, step {step}"))?; + if predicted < 0 { + bail!("continuation decode returned no token at width {width}, step {step}"); + } + predicted_tokens.push(predicted); + input = predicted; + } + Ok(predicted_tokens) +} + +fn prediction_prefix(prediction: &[i32], width: usize) -> Result<&[i32]> { + if prediction.len() < width { + bail!( + "prediction has {} token(s), but width {width} requires {width}", + prediction.len() + ); + } + Ok(&prediction[..width]) +} + +fn first_mismatch_position(left: &[i32], right: &[i32]) -> Option { + left.iter() + .zip(right.iter()) + .position(|(left, right)| left != right) + .or_else(|| (left.len() != right.len()).then_some(left.len().min(right.len()))) +} + +fn max_verify_width(args: &VerifyWindowLocalArgs) -> Result { + args.verify_widths + .iter() .copied() - .context("native MTP draft did not include a token")?; - Ok(vec![current, draft_token]) + .max() + .context("verify_widths must contain at least one width") +} + +fn target_token_count(args: &VerifyWindowLocalArgs) -> Result { + max_verify_width(args)? + .max(args.sample_width) + .checked_add(args.continuation_steps) + .context("target token count overflow") +} + +fn sample_width(args: &VerifyWindowLocalArgs) -> usize { + args.sample_width } fn run_samples( @@ -364,7 +781,7 @@ fn measure_batched( let start = Instant::now(); let prediction = session .verify_tokens_frame_sampled(verify_tokens, Some(&SamplingConfig::default()), None, 0) - .context("batched width-2 VerifyWindow failed")? + .with_context(|| format!("batched width-{} VerifyWindow failed", verify_tokens.len()))? .0; Ok((start.elapsed(), prediction)) } @@ -412,7 +829,12 @@ fn run_split_inprocess_samples( args.warmup, args.iterations, )?; - split_report(split_layer, samples, full_batched_avg_us) + split_report( + split_layer, + samples, + full_batched_avg_us, + verify_tokens.len(), + ) } fn split_runtime_configs( @@ -437,7 +859,7 @@ fn split_runtime_configs( selected_backend_device: None, cache_type_k, cache_type_v, - flash_attn_type: FlashAttentionType::Auto, + flash_attn_type: runtime_flash_attn(args.flash_attn), load_mode: RuntimeLoadMode::RuntimeSlice, projector_path: None, include_embeddings: true, @@ -460,7 +882,7 @@ fn split_runtime_configs( selected_backend_device: None, cache_type_k, cache_type_v, - flash_attn_type: FlashAttentionType::Auto, + flash_attn_type: runtime_flash_attn(args.flash_attn), load_mode: RuntimeLoadMode::RuntimeSlice, projector_path: None, include_embeddings: false, @@ -707,6 +1129,7 @@ fn split_report( split_layer: u32, samples: SplitSampleSet, full_batched_avg_us: f64, + sample_width: usize, ) -> Result { let total = timing_stats(&samples.total)?; let serial_total = timing_stats(&samples.serial_total)?; @@ -722,8 +1145,8 @@ fn split_report( serial_stage1: timing_stats(&samples.serial_stage1)?, total, serial_total, - total_token_per_sec: verified_tokens_per_sec(total_avg, 2), - serial_total_token_per_sec: verified_tokens_per_sec(serial_total_avg, 2), + total_token_per_sec: verified_tokens_per_sec(total_avg, sample_width), + serial_total_token_per_sec: verified_tokens_per_sec(serial_total_avg, sample_width), total_avg_vs_full_batched_avg: total_avg / full_batched_avg_us, total_avg_vs_serial_total_avg: total_avg / serial_total_avg, diagnostics: split_timing_diagnostics(&samples)?, @@ -760,12 +1183,13 @@ impl SampleSet { fn build_report( args: VerifyWindowLocalArgs, full: FullModelSamples, - split_inprocess_width2: Option, + split_inprocess: Option, ) -> Result { - let batched_width2 = timing_stats(&full.samples.batched)?; - let serial_two_decode_mtp_n1 = timing_stats(&full.samples.serial)?; - let batched_avg = batched_width2.avg_us; - let serial_avg = serial_two_decode_mtp_n1.avg_us; + let sample_width = sample_width(&args); + let batched = timing_stats(&full.samples.batched)?; + let serial = timing_stats(&full.samples.serial)?; + let batched_avg = batched.avg_us; + let serial_avg = serial.avg_us; Ok(VerifyWindowLocalReport { mode: "verify-window-local", model_path: args.model_path, @@ -777,16 +1201,21 @@ fn build_report( n_ubatch: args.n_ubatch, cache_type_k: args.cache_type_k, cache_type_v: args.cache_type_v, + flash_attn: flash_attn_name(args.flash_attn), prompt_token_count: full.tokens.len(), verify_tokens: full.verify_tokens, + verify_widths: args.verify_widths, + continuation_steps: args.continuation_steps, + parity_checks: full.parity_checks, warmup: args.warmup, iterations: args.iterations, - batched_width2, - serial_two_decode_mtp_n1, - split_inprocess_width2, + sample_width, + batched, + serial, + split_inprocess, batched_avg_vs_serial_avg: batched_avg / serial_avg, - batched_token_per_sec: verified_tokens_per_sec(batched_avg, 2), - serial_token_per_sec: verified_tokens_per_sec(serial_avg, 2), + batched_token_per_sec: verified_tokens_per_sec(batched_avg, sample_width), + serial_token_per_sec: verified_tokens_per_sec(serial_avg, sample_width), first_batched_prediction: full.samples.first_batched_prediction, first_serial_prediction: full.samples.first_serial_prediction, }) @@ -852,11 +1281,34 @@ fn model_description(config: &RuntimeConfig) -> String { ) } +fn runtime_flash_attn(value: FlashAttentionArg) -> FlashAttentionType { + match value { + FlashAttentionArg::Auto => FlashAttentionType::Auto, + FlashAttentionArg::Disabled => FlashAttentionType::Disabled, + FlashAttentionArg::Enabled => FlashAttentionType::Enabled, + } +} + +fn flash_attn_name(value: FlashAttentionArg) -> &'static str { + match value { + FlashAttentionArg::Auto => "auto", + FlashAttentionArg::Disabled => "disabled", + FlashAttentionArg::Enabled => "enabled", + } +} + #[cfg(test)] mod tests { use std::time::Duration; - use super::{percentile, timing_shape, timing_stats, verified_tokens_per_sec}; + use skippy_runtime::{GenerationSignalWindow, TokenSignal}; + + use super::{ + ParitySide, VerifyTargetPlan, build_parity_check, expected_continuation, + first_mismatch_position, percentile, timing_shape, timing_stats, validate_verify_widths, + verified_tokens_per_sec, + }; + use crate::cli::MAX_VERIFY_WINDOW_WIDTH; #[test] fn timing_stats_sorts_and_summarizes_microseconds() { @@ -880,8 +1332,17 @@ mod tests { } #[test] - fn token_rate_uses_verified_token_count() { - assert_eq!(verified_tokens_per_sec(20_000.0, 2), 100.0); + fn token_rate_uses_configured_verify_width() { + assert_eq!(verified_tokens_per_sec(20_000.0, 4), 200.0); + } + + #[test] + fn verify_width_validation_rejects_duplicates_and_out_of_range_samples() { + assert!(validate_verify_widths(&[1, 2, 4, 9, 16], 9).is_ok()); + assert!(validate_verify_widths(&[1, 2, 2], 2).is_err()); + assert!(validate_verify_widths(&[MAX_VERIFY_WINDOW_WIDTH + 1], 2).is_err()); + assert!(validate_verify_widths(&[2], MAX_VERIFY_WINDOW_WIDTH + 1).is_err()); + assert!(validate_verify_widths(&[1, 2, 4], 9).is_err()); } #[test] @@ -901,4 +1362,93 @@ mod tests { assert_eq!(shape.second_half.avg_us, 40.0); assert_eq!(shape.second_half_avg_vs_first_half_avg, 40.0 / 15.0); } + + #[test] + fn target_plan_builds_required_verification_widths() { + let plan = VerifyTargetPlan { + current: 10, + targets: (20..=40).collect(), + }; + + for width in [1, 2, 4, 9] { + let tokens = plan.verify_tokens_for_width(width).unwrap(); + let expected = plan.expected_for_width(width).unwrap(); + + assert_eq!(tokens.len(), width); + assert_eq!(expected.len(), width); + assert_eq!(tokens[0], 10); + assert_eq!(&tokens[1..], &expected[..width.saturating_sub(1)]); + } + } + + #[test] + fn target_plan_selects_expected_continuation_after_width() { + let plan = VerifyTargetPlan { + current: 10, + targets: (20..=40).collect(), + }; + + assert_eq!(expected_continuation(&plan, 4, 3).unwrap(), &[24, 25, 26]); + } + + #[test] + fn parity_aggregation_preserves_mismatch_details() { + let serial = parity_side(vec![11, 12], vec![13], GenerationSignalWindow::default()); + let batched = parity_side(vec![11, 99], vec![13], GenerationSignalWindow::default()); + + let check = build_parity_check(2, 1, &[11, 12], &[13], &serial, &batched).unwrap(); + + assert!(!check.matched); + assert_eq!(check.first_mismatch_position, Some(1)); + assert!(!check.target_stream_matched); + assert!(check.continuation_matched); + } + + #[test] + fn signal_window_shape_is_advisory_for_batch_parity() { + let serial = parity_side( + vec![11, 12], + vec![13], + GenerationSignalWindow { + token_count: 2, + ..GenerationSignalWindow::default() + }, + ); + let batched = parity_side( + vec![11, 12], + vec![13], + GenerationSignalWindow { + token_count: 1, + ..GenerationSignalWindow::default() + }, + ); + + let check = build_parity_check(2, 1, &[11, 12], &[13], &serial, &batched).unwrap(); + + assert!(check.matched); + assert!(!check.signal_window_matched); + } + + fn parity_side( + prediction: Vec, + continuation: Vec, + signal_window: GenerationSignalWindow, + ) -> ParitySide { + ParitySide { + prediction, + continuation, + position: 2, + full_state: vec![1, 2, 3], + signal: TokenSignal::default(), + signal_window, + elapsed_us: 1, + } + } + + #[test] + fn mismatch_position_reports_value_and_length_differences() { + assert_eq!(first_mismatch_position(&[1, 2, 3], &[1, 9, 3]), Some(1)); + assert_eq!(first_mismatch_position(&[1, 2], &[1, 2, 3]), Some(2)); + assert_eq!(first_mismatch_position(&[1, 2, 3], &[1, 2, 3]), None); + } } diff --git a/third_party/llama.cpp/patches/0043-ggml-metal-make-small-batch-matmul-batch-invariant.patch b/third_party/llama.cpp/patches/0043-ggml-metal-make-small-batch-matmul-batch-invariant.patch new file mode 100644 index 0000000000..28276aae2a --- /dev/null +++ b/third_party/llama.cpp/patches/0043-ggml-metal-make-small-batch-matmul-batch-invariant.patch @@ -0,0 +1,360 @@ +From 0f7f31ef18355e7581e0e5a36f5a45e45274b3aa Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sun, 26 Jul 2026 11:28:36 +1000 +Subject: [PATCH] ggml-metal: make small-batch matmul batch invariant + +--- + ggml/src/ggml-metal/ggml-metal-device.cpp | 8 +- + ggml/src/ggml-metal/ggml-metal-impl.h | 2 + + ggml/src/ggml-metal/ggml-metal-ops.cpp | 108 +++++++++++++++++----- + ggml/src/ggml-metal/ggml-metal.metal | 103 +++++++++++++++++---- + 4 files changed, 176 insertions(+), 45 deletions(-) + +diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp +index 870d5a92e..b156d670c 100644 +--- a/ggml/src/ggml-metal/ggml-metal-device.cpp ++++ b/ggml/src/ggml-metal/ggml-metal-device.cpp +@@ -1186,6 +1186,12 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_ + ggml_metal_cv_free(cv); + } + ++ if (tsrc0 == GGML_TYPE_F32 && tsrc1 == GGML_TYPE_F32) { ++ res.smem = 32*N_R0_F32*sizeof(float); ++ } else if (tsrc0 == GGML_TYPE_Q5_0 && tsrc1 == GGML_TYPE_F32) { ++ res.smem = 32*N_R0_Q5_0*sizeof(float); ++ } ++ + return res; + } + +@@ -1282,7 +1288,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta + suffix = "_short"; + } else { + nsg = std::min(4, (ne00 + 127) / 128); +- nr0 = 2; ++ nr0 = N_R0_F32; + nr1 = 1; + smem = 32*sizeof(float)*nr0; + suffix = ne00 % 4 == 0 ? "_4" : ""; +diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h +index f7112ee56..b8683e534 100644 +--- a/ggml/src/ggml-metal/ggml-metal-impl.h ++++ b/ggml/src/ggml-metal/ggml-metal-impl.h +@@ -44,6 +44,8 @@ + #define N_R0_Q8_0 2 + #define N_SG_Q8_0 4 + ++#define N_R0_F32 2 ++ + #define N_R0_MXFP4 2 + #define N_SG_MXFP4 2 + +diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp +index 646c983ad..a4b62a42e 100644 +--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp ++++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp +@@ -7704,6 +7704,41 @@ int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { + return 1; + } + ++struct ggml_metal_mul_mv_ext_row_params { ++ int nsg; ++ int16_t r0ptg; ++}; ++ ++static constexpr int64_t GGML_METAL_BATCH_INVARIANT_MUL_MV_MAX_WIDTH = 16; ++ ++static ggml_metal_mul_mv_ext_row_params ggml_metal_mul_mv_ext_row_params_for( ++ enum ggml_type type, ++ int64_t ne00, ++ int16_t nypsg, ++ bool batch_invariant) { ++ // Keep the F32 geometry aligned with ggml_metal_library_get_pipeline_mul_mv(). ++ const int nsg = type == GGML_TYPE_F32 ? std::min(4, (int) ((ne00 + 127)/128)) : ++ type == GGML_TYPE_Q5_0 ? N_SG_Q5_0 : ++ type == GGML_TYPE_Q4_K ? N_SG_Q4_K : ++ type == GGML_TYPE_Q6_K ? N_SG_Q6_K : 2; ++ if (!batch_invariant) { ++ return { nsg, (int16_t) (nypsg*nsg) }; ++ } ++ ++ switch (type) { ++ case GGML_TYPE_F32: ++ return { nsg, N_R0_F32 }; ++ case GGML_TYPE_Q5_0: ++ return { nsg, (int16_t) (N_R0_Q5_0*nsg) }; ++ case GGML_TYPE_Q4_K: ++ return { nsg, (int16_t) (N_R0_Q4_K*nsg) }; ++ case GGML_TYPE_Q6_K: ++ return { nsg, (int16_t) (N_R0_Q6_K*nsg) }; ++ default: ++ GGML_ABORT("unsupported batch-invariant mul_mv_ext type"); ++ } ++} ++ + int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + +@@ -7817,10 +7852,17 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + // find the break-even point where the matrix-matrix kernel becomes more efficient compared + // to the matrix-vector kernel + const int ne11_mm_min = 8; ++ const bool batch_invariant_ext_type = ++ op->src[0]->type == GGML_TYPE_F32 || ++ op->src[0]->type == GGML_TYPE_Q5_0 || ++ op->src[0]->type == GGML_TYPE_Q4_K || ++ op->src[0]->type == GGML_TYPE_Q6_K; + + // first try to use small-batch mat-mv kernels + // these should be efficient for BS [2, ~8] +- if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) && ++ if (op->src[1]->type == GGML_TYPE_F32 && ++ (ne00%128 == 0 || ++ (op->src[0]->type == GGML_TYPE_Q5_0 && ne00%32 == 0)) && + ( + ( + ( +@@ -7836,7 +7878,10 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_MXFP4 || + op->src[0]->type == GGML_TYPE_IQ4_NL || +- false) && (ne11 >= 2 && ne11 <= 8) ++ false) && (ne11 >= 2 && ne11 <= ++ ((op->src[0]->type == GGML_TYPE_F32 || ++ op->src[0]->type == GGML_TYPE_Q5_0) ? ++ GGML_METAL_BATCH_INVARIANT_MUL_MV_MAX_WIDTH : 8)) + ) || + ( + ( +@@ -7845,7 +7890,10 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_Q2_K || + op->src[0]->type == GGML_TYPE_Q3_K || +- false) && (ne11 >= 4 && ne11 <= 8) ++ false) && (ne11 >= 4 && ne11 <= ++ ((op->src[0]->type == GGML_TYPE_Q4_K || ++ op->src[0]->type == GGML_TYPE_Q6_K) ? ++ GGML_METAL_BATCH_INVARIANT_MUL_MV_MAX_WIDTH : 8)) + ) + ) + ) { +@@ -7857,8 +7905,6 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + // my current hypothesis is that the work grid is not evenly divisible for different nsg + // values and there can be some tail effects when nsg is high. need to confirm this + // +- const int nsg = 2; // num simdgroups per threadgroup +- + // num threads along row per simdgroup + int16_t nxpsg = 0; + if (ne00 % 256 == 0 && ne11 < 3) { +@@ -7869,26 +7915,33 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + nxpsg = 4; + } + +- const int16_t nypsg = 32/nxpsg; // num threads along col per simdgroup (i.e. a simdgroup processes that many src0 rows at a time) +- const int16_t r0ptg = nypsg*nsg; // num src0 rows per threadgroup +- int16_t r1ptg = 4; // num src1 rows per threadgroup ++ const int16_t nypsg = 32/nxpsg; // num threads along col per simdgroup (i.e. a simdgroup processes that many src0 rows at a time) ++ const auto row_params = ggml_metal_mul_mv_ext_row_params_for( ++ op->src[0]->type, ne00, nypsg, batch_invariant_ext_type); ++ const int nsg = row_params.nsg; // num simdgroups per threadgroup ++ const int16_t r0ptg = row_params.r0ptg; // num src0 rows per threadgroup ++ // Batch-invariant kernels ignore r1ptg and expose only the _r1_2 pipeline. ++ // Other ext kernels retain the canonical rows-per-threadgroup selection. ++ int16_t r1ptg = 2; + + // note: not sure how optimal are those across all different hardware. there might be something cleverer +- switch (ne11) { +- case 2: +- r1ptg = 2; break; +- case 3: +- case 6: +- r1ptg = 3; break; +- case 4: +- case 7: +- case 8: +- r1ptg = 4; break; +- case 5: +- r1ptg = 5; break; +- default: +- GGML_ABORT("unsupported ne11"); +- }; ++ if (!batch_invariant_ext_type) { ++ switch (ne11) { ++ case 2: ++ r1ptg = 2; break; ++ case 3: ++ case 6: ++ r1ptg = 3; break; ++ case 4: ++ case 7: ++ case 8: ++ r1ptg = 4; break; ++ case 5: ++ r1ptg = 5; break; ++ default: ++ GGML_ABORT("unsupported ne11"); ++ }; ++ } + + auto pipeline = ggml_metal_library_get_pipeline_mul_mv_ext(lib, op, nsg, nxpsg, r1ptg); + +@@ -7919,7 +7972,14 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + +- ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1); ++ if (pipeline.smem > 0) { ++ ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); ++ } ++ ++ const int64_t grid_y = batch_invariant_ext_type ? ++ ne11 : (ne11 + r1ptg - 1)/r1ptg; ++ ggml_metal_encoder_dispatch_threadgroups( ++ enc, (ne01 + r0ptg - 1)/r0ptg, grid_y, ne12*ne13, 32, nsg, 1); + } else if ( + !ggml_is_transposed(op->src[0]) && + !ggml_is_transposed(op->src[1]) && +diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal +index b098231dc..4e6d2b35f 100644 +--- a/ggml/src/ggml-metal/ggml-metal.metal ++++ b/ggml/src/ggml-metal/ggml-metal.metal +@@ -4778,11 +4778,6 @@ kernel void kernel_mul_mv_ext_q4x4_f32_disp( + typedef decltype(kernel_mul_mv_ext_q4_f32_disp <2, block_q8_0, 32, dequantize_q8_0_t4>) mul_mv_ext_q4_f32_t; + typedef decltype(kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>) mul_mv_ext_q4x4_f32_t; + +-template [[host_name("kernel_mul_mv_ext_f32_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, float4, 4, dequantize_f32_t4>; +-template [[host_name("kernel_mul_mv_ext_f32_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, float4, 4, dequantize_f32_t4>; +-template [[host_name("kernel_mul_mv_ext_f32_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, float4, 4, dequantize_f32_t4>; +-template [[host_name("kernel_mul_mv_ext_f32_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, float4, 4, dequantize_f32_t4>; +- + template [[host_name("kernel_mul_mv_ext_f16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, half4, 4, dequantize_f16_t4>; + template [[host_name("kernel_mul_mv_ext_f16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, half4, 4, dequantize_f16_t4>; + template [[host_name("kernel_mul_mv_ext_f16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, half4, 4, dequantize_f16_t4>; +@@ -4815,11 +4810,6 @@ template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_3")]] kernel mul_mv_ext_q4 + template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_1, 32, dequantize_q4_1_t4>; + template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_1, 32, dequantize_q4_1_t4>; + +-template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_0, 32, dequantize_q5_0_t4>; +-template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_0, 32, dequantize_q5_0_t4>; +-template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_0, 32, dequantize_q5_0_t4>; +-template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_0, 32, dequantize_q5_0_t4>; +- + template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_1, 32, dequantize_q5_1_t4>; + template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_1, 32, dequantize_q5_1_t4>; + template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_1, 32, dequantize_q5_1_t4>; +@@ -4840,21 +4830,11 @@ template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_3")]] kernel mul_mv_ext_q4 + template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_iq4_nl, 32, dequantize_iq4_nl_t4>; + template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_iq4_nl, 32, dequantize_iq4_nl_t4>; + +-template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>; +-template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q4_K, 256, dequantize_q4_K>; +-template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q4_K, 256, dequantize_q4_K>; +-template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q4_K, 256, dequantize_q4_K>; +- + template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q5_K, 256, dequantize_q5_K>; + template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q5_K, 256, dequantize_q5_K>; + template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q5_K, 256, dequantize_q5_K>; + template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q5_K, 256, dequantize_q5_K>; + +-template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q6_K, 256, dequantize_q6_K>; +-template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q6_K, 256, dequantize_q6_K>; +-template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q6_K, 256, dequantize_q6_K>; +-template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q6_K, 256, dequantize_q6_K>; +- + template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>; + template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>; + template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>; +@@ -13429,6 +13409,89 @@ kernel void kernel_mul_mv_q6_K_f32( + kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); + } + ++// Batch-invariant small-batch kernels. Each input row uses the same arithmetic ++// and reduction tree as the canonical single-row mul_mv implementation. ++template ++kernel void kernel_mul_mv_ext_f32_f32_batch_invariant( ++ constant ggml_metal_kargs_mul_mv_ext & args, ++ device const char * src0, ++ device const char * src1, ++ device char * dst, ++ threadgroup char * shmem [[threadgroup(0)]], ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ (void) r1ptg; ++ kernel_mul_mv_t_t_4_impl< ++ float, float4, float, float4, 2, ++ constant ggml_metal_kargs_mul_mv_ext &>( ++ args, src0, src1, dst, shmem, ++ tgpig, tiisg, sgitg); ++} ++ ++template ++kernel void kernel_mul_mv_ext_q4_K_f32_batch_invariant( ++ constant ggml_metal_kargs_mul_mv_ext & args, ++ device const char * src0, ++ device const char * src1, ++ device char * dst, ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ (void) r1ptg; ++ kernel_mul_mv_q4_K_f32_impl< ++ N_R0_Q4_K, constant ggml_metal_kargs_mul_mv_ext &>( ++ args, src0, src1, dst, nullptr, ++ tgpig, tiisg, sgitg); ++} ++ ++template ++kernel void kernel_mul_mv_ext_q6_K_f32_batch_invariant( ++ constant ggml_metal_kargs_mul_mv_ext & args, ++ device const char * src0, ++ device const char * src1, ++ device char * dst, ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ (void) r1ptg; ++ kernel_mul_mv_q6_K_f32_impl< ++ N_R0_Q6_K, constant ggml_metal_kargs_mul_mv_ext &>( ++ args, src0, src1, dst, nullptr, ++ tgpig, tiisg, sgitg); ++} ++ ++template ++kernel void kernel_mul_mv_ext_q5_0_f32_batch_invariant( ++ constant ggml_metal_kargs_mul_mv_ext & args, ++ device const char * src0, ++ device const char * src1, ++ device char * dst, ++ threadgroup char * shmem [[threadgroup(0)]], ++ uint3 tgpig[[threadgroup_position_in_grid]], ++ ushort tiisg[[thread_index_in_simdgroup]], ++ ushort sgitg[[simdgroup_index_in_threadgroup]]) { ++ (void) r1ptg; ++ mul_vec_q_n_f32_impl< ++ block_q5_0, N_R0_Q5_0, ++ constant ggml_metal_kargs_mul_mv_ext &>( ++ args, src0, src1, dst, shmem, ++ tgpig, tiisg, sgitg); ++} ++ ++typedef decltype(kernel_mul_mv_ext_f32_f32_batch_invariant<2>) mul_mv_ext_f32_f32_batch_invariant_t; ++typedef decltype(kernel_mul_mv_ext_q4_K_f32_batch_invariant<2>) mul_mv_ext_q4_K_f32_batch_invariant_t; ++typedef decltype(kernel_mul_mv_ext_q6_K_f32_batch_invariant<2>) mul_mv_ext_q6_K_f32_batch_invariant_t; ++typedef decltype(kernel_mul_mv_ext_q5_0_f32_batch_invariant<2>) mul_mv_ext_q5_0_f32_batch_invariant_t; ++ ++template [[host_name("kernel_mul_mv_ext_f32_f32_r1_2")]] kernel mul_mv_ext_f32_f32_batch_invariant_t kernel_mul_mv_ext_f32_f32_batch_invariant<2>; ++ ++template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_2")]] kernel mul_mv_ext_q4_K_f32_batch_invariant_t kernel_mul_mv_ext_q4_K_f32_batch_invariant<2>; ++ ++template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_2")]] kernel mul_mv_ext_q6_K_f32_batch_invariant_t kernel_mul_mv_ext_q6_K_f32_batch_invariant<2>; ++ ++template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_2")]] kernel mul_mv_ext_q5_0_f32_batch_invariant_t kernel_mul_mv_ext_q5_0_f32_batch_invariant<2>; ++ + // ======================= "True" 2-bit + + template +-- +2.54.0 (Apple Git-157) +