Expand L4/L5 test coverage: golden data, seq2seq generation, ci_skip_reason

scripts/arch_diff.py

@@ -220,20 +220,8 @@
         "text-generation",
         "standard",
     ),
-    (
-        "gemma4_text",
-        {
-            "_config_cls": "Gemma4Config",
-            "attn_qk_norm": True,
-            "rope_local_base_freq": 10_000.0,
-            "layer_types": ["sliding_attention", "full_attention"],
-            "global_head_dim": 16,
-            "global_rope_theta": 10_000.0,
-            "final_logit_softcapping": 30.0,
-        },
-        "static-cache",
-        "standard",
-    ),
+    # NOTE: gemma4_text does not support static-cache mode because
+    # Gemma4DecoderLayer inherits from nn.Module, not DecoderLayer.
     ("gemma4", {}, "gemma4", "gemma4"),
     ("qwen3_5_vl", {}, "hybrid-qwen-vl", "qwen3_5_vl"),
     ("whisper", {}, "speech-to-text", "whisper"),

scripts/generate_dashboard.py

@@ -83,6 +83,7 @@ class ModelInfo:
     l5_test_case_skipped: bool = False
     yaml_test_case_file: str | None = None
     yaml_test_case_skip_reason: str | None = None
+    yaml_test_case_ci_skip_reason: str | None = None
     yaml_min_token_match_ratio: float | None = None
     # L3 synthetic parity status: "pass", "xfail", "skip", or None
     l3_status: str | None = None
@@ -413,6 +414,7 @@ def _scan_yaml_test_cases(models: dict[str, ModelInfo]) -> None:
         # Skip test cases that are explicitly skipped — they don't count as coverage,
         # but we still record them so the dashboard can show "skipped" status.
         skip_reason = data.get("skip_reason")
+        ci_skip_reason = data.get("ci_skip_reason")
         min_token_match_ratio = data.get("min_token_match_ratio")
         if skip_reason:
             matched_types = model_id_to_types.get(model_id, [])
@@ -435,6 +437,8 @@ def _scan_yaml_test_cases(models: dict[str, ModelInfo]) -> None:
         for model_type in matched_types:
             if model_type in models:
                 models[model_type].yaml_test_case_file = rel_path
+                if ci_skip_reason:
+                    models[model_type].yaml_test_case_ci_skip_reason = ci_skip_reason
                 if min_token_match_ratio is not None:
                     models[model_type].yaml_min_token_match_ratio = float(
                         min_token_match_ratio
@@ -711,6 +715,7 @@ def _render_html(
                 "l3_reason": info.l3_status_reason,
                 "yaml_case": info.yaml_test_case_file,
                 "yaml_skip_reason": info.yaml_test_case_skip_reason,
+                "yaml_ci_skip_reason": info.yaml_test_case_ci_skip_reason,
                 "min_token_match_ratio": info.yaml_min_token_match_ratio,
                 "code_paths": sorted(info.code_paths),
                 "config_overrides": _json_safe(info.config_overrides),

scripts/generate_golden.py

@@ -230,6 +230,15 @@ def _generate_encoder(case: TestCase, json_path: Path, device: str) -> None:
 
     model, tokenizer = load_torch_encoder_model(case.model_id, device=device)
 
+    # CLIP-like multimodal models wrap a text sub-model that can be
+    # called with text-only inputs (pixel_values not required).
+    if hasattr(model, "text_model"):
+        model = model.text_model
+
+    # X-MOD requires setting a default language before inference.
+    if hasattr(model, "set_default_language"):
+        model.set_default_language("en_XX")
+
     encoded = tokenizer(case.prompts[0], return_tensors="np", padding=False)
     input_ids = encoded["input_ids"]
     attention_mask = encoded["attention_mask"]
@@ -267,7 +276,11 @@ def _generate_seq2seq(case: TestCase, json_path: Path, device: str) -> None:
     input_ids = encoded["input_ids"]
 
     # Prepare decoder input (pad token for autoregressive start)
-    decoder_start = np.array([[model.config.decoder_start_token_id or 0]], dtype=np.int64)
+    decoder_start_id = getattr(model.config, "decoder_start_token_id", None)
+    if decoder_start_id is None:
+        generation_config = getattr(model, "generation_config", None)
+        decoder_start_id = getattr(generation_config, "decoder_start_token_id", None) or 0
+    decoder_start = np.array([[decoder_start_id]], dtype=np.int64)
 
     # L4: single forward pass through full model
     torch_device = next(model.parameters()).device
@@ -635,12 +648,18 @@ def _prepare_speech_language_inputs(
         text_prompt = processor.apply_chat_template(
             messages, add_generation_prompt=True, tokenize=False
         )
+        # If prompts are provided, append as forced decoder prefix
+        # (e.g. "language English<asr_text>" to skip language detection).
+        force_prefix = ""
+        if case.prompts:
+            force_prefix = case.prompts[0]
+            text_prompt = text_prompt + force_prefix
         processed = processor(
             text=text_prompt,
             audio=[audio_array],
             return_tensors="pt",
         ).to(device)
-        prompt_for_golden = str(audio_path)
+        prompt_for_golden = force_prefix or str(audio_path)
     else:
         # Gemma4-style: text prompt + audio
         prompt_text = case.prompts[0]
@@ -735,10 +754,20 @@ def _generate_image_classification(case: TestCase, json_path: Path, device: str)
 
     # Forward pass → last_hidden_state
     hidden_states = torch_vision_forward(model, pixel_values)
-    # Use the last patch token rather than the CLS token (index 0) because
-    # patch-based ViT models aggregate spatial context into trailing tokens;
-    # the last token provides a stable, architecture-neutral summary vector.
-    last_hidden = hidden_states[0, -1, :]  # (hidden_size,)
+    # Vision models return different output shapes:
+    # - ViT-like: [B, seq_len, hidden] → select first token (CLS)
+    # - CNN-like (CvT, MobileViT, PVT): [B, C, H, W] → flatten feature map
+    # - Classification head: [B, num_classes] → 1-D logits
+    batch_hidden = hidden_states[0]  # drop batch dim
+    if batch_hidden.ndim == 2:
+        # (seq_len, hidden) — take CLS token
+        last_hidden = batch_hidden[0]
+    elif batch_hidden.ndim >= 3:
+        # (C, H, W) feature map — flatten
+        last_hidden = batch_hidden.reshape(-1)
+    else:
+        # 1-D logits or already flat
+        last_hidden = batch_hidden
     golden = _extract_logits_golden(last_hidden)
 
     # Image classification is L4-only (no generation)
@@ -765,6 +794,11 @@ def _generate_image_classification(case: TestCase, json_path: Path, device: str)
     "speech-to-text": _generate_speech_to_text,
     "speech-language": _generate_speech_language,
     "audio-feature-extraction": _generate_audio_feature_extraction,
+    # Vision tasks that produce last_hidden_state — reuse image classification.
+    "depth-estimation": _generate_image_classification,
+    "image-segmentation": _generate_image_classification,
+    "image-to-image": _generate_image_classification,
+    "object-detection": _generate_image_classification,
 }
 
 

src/mobius/_configs.py

@@ -271,6 +271,7 @@ class Gemma4AudioConfig(AudioConfig):
     hidden_size: int = 1024
     subsampling_conv_channels: list[int] | None = None
     use_causal_chunked_attn: bool = False
+    output_proj_dims: int | None = None
 
 
 def _first_not_none(*values, default=None):
@@ -627,6 +628,7 @@ def _extract_audio_config(config, parent_config, model_type: str) -> dict:
                     ),
                     use_causal_chunked_attn=getattr(ac, "use_causal_chunked_attn", False),
                     output_dim=getattr(ac, "output_dim", None),
+                    output_proj_dims=getattr(ac, "output_proj_dims", None),
                     audio_token_id=getattr(composite, "audio_token_id", None),
                 )
             }