Expand L4/L5 test coverage: golden data, seq2seq generation, ci_skip_reason

scripts/generate_golden.py

@@ -230,6 +230,15 @@ def _generate_encoder(case: TestCase, json_path: Path, device: str) -> None:
 
     model, tokenizer = load_torch_encoder_model(case.model_id, device=device)
 
+    # CLIP-like multimodal models wrap a text sub-model that can be
+    # called with text-only inputs (pixel_values not required).
+    if hasattr(model, "text_model"):
+        model = model.text_model
+
+    # X-MOD requires setting a default language before inference.
+    if hasattr(model, "set_default_language"):
+        model.set_default_language("en_XX")
+
     encoded = tokenizer(case.prompts[0], return_tensors="np", padding=False)
     input_ids = encoded["input_ids"]
     attention_mask = encoded["attention_mask"]
@@ -267,7 +276,10 @@ def _generate_seq2seq(case: TestCase, json_path: Path, device: str) -> None:
     input_ids = encoded["input_ids"]
 
     # Prepare decoder input (pad token for autoregressive start)
-    decoder_start = np.array([[model.config.decoder_start_token_id or 0]], dtype=np.int64)
+    decoder_start_id = getattr(model.config, "decoder_start_token_id", None)
+    if decoder_start_id is None:
+        decoder_start_id = getattr(model.generation_config, "decoder_start_token_id", 0) or 0
+    decoder_start = np.array([[decoder_start_id]], dtype=np.int64)
 
     # L4: single forward pass through full model
     torch_device = next(model.parameters()).device
@@ -735,10 +747,11 @@ def _generate_image_classification(case: TestCase, json_path: Path, device: str)
 
     # Forward pass → last_hidden_state
     hidden_states = torch_vision_forward(model, pixel_values)
-    # Use the last patch token rather than the CLS token (index 0) because
-    # patch-based ViT models aggregate spatial context into trailing tokens;
-    # the last token provides a stable, architecture-neutral summary vector.
-    last_hidden = hidden_states[0, -1, :]  # (hidden_size,)
+    # Some vision models (CvT, MobileViT, PVT) return 4-D feature maps
+    # [B, C, H, W] instead of [B, seq_len, hidden].  Flatten everything
+    # into a 1-D vector for top-k extraction.
+    batch_hidden = hidden_states[0]  # drop batch dim
+    last_hidden = batch_hidden.reshape(-1) if batch_hidden.ndim > 1 else batch_hidden
     golden = _extract_logits_golden(last_hidden)
 
     # Image classification is L4-only (no generation)
@@ -765,6 +778,11 @@ def _generate_image_classification(case: TestCase, json_path: Path, device: str)
     "speech-to-text": _generate_speech_to_text,
     "speech-language": _generate_speech_language,
     "audio-feature-extraction": _generate_audio_feature_extraction,
+    # Vision tasks that produce last_hidden_state — reuse image classification.
+    "depth-estimation": _generate_image_classification,
+    "image-segmentation": _generate_image_classification,
+    "image-to-image": _generate_image_classification,
+    "object-detection": _generate_image_classification,
 }
 
 

scripts/templates/dashboard.html.j2

@@ -384,9 +384,17 @@ const LEVEL_DESCRIPTIONS = {
   bar.innerHTML = totalCard;
   for (let i = 0; i <= 5; i++) {
     const count = SUMMARY.by_level[i] || 0;
-    // Annotate L4 and L5 cards with skipped + awaiting-data counts so the full
-    // picture is visible without a separate redundant card.
+    // Annotate level cards with status breakdowns.
     let annotation = '';
+    if (i === 3) {
+      // L3: show pass/xfail/skip parity status breakdown
+      const l3s = SUMMARY.l3_status_counts || {};
+      const parts = [];
+      if (l3s.pass) parts.push(`<span style="color:var(--l3)">${l3s.pass}\u2713</span>`);
+      if (l3s.xfail) parts.push(`<span style="color:var(--l1)">${l3s.xfail}\u26A0</span>`);
+      if (l3s.skip) parts.push(`<span style="color:var(--text-muted)">${l3s.skip}\u23ED</span>`);
+      if (parts.length) annotation = `<div class="card-annotation">${parts.join(' ')}</div>`;
+    }
     if (i === 4 || i === 5) {
       const skipped = i === 4 ? (SUMMARY.l4_skipped_count || 0) : (SUMMARY.l5_skipped_count || 0);
       const caseKey = i === 4 ? 'l4_case' : 'l5_case';
@@ -403,19 +411,7 @@ const LEVEL_DESCRIPTIONS = {
       ${annotation}
     </div>`;
   }
-  // L3 parity breakdown
-  const l3s = SUMMARY.l3_status_counts || {};
-  if (l3s.pass || l3s.xfail || l3s.skip) {
-    bar.innerHTML += `<div class="summary-card" title="L3 parity status breakdown">
-      <div class="number" style="font-size:1em;line-height:1.4">
-        <span style="color:var(--l3)">${l3s.pass || 0}\u2713</span>
-        <span style="color:var(--l1)">${l3s.xfail || 0}\u26A0</span>
-        <span style="color:var(--text-muted)">${l3s.skip || 0}\u23ED</span>
-      </div>
-      <div class="label">L3 Parity Status</div>
-    </div>`;
-  }
-  // (L4/L5 skipped counts are shown as annotations on the L4/L5 level cards above.)
+  // (L3 parity status and L4/L5 skipped counts are shown as annotations on their level cards above.)
 })();
 
 // --- Category filter ---

src/mobius/_testing/torch_reference.py

@@ -374,12 +374,33 @@ def load_torch_vision_model(
     processor = transformers.AutoImageProcessor.from_pretrained(
         model_id, trust_remote_code=trust_remote_code
     )
-    model = transformers.AutoModel.from_pretrained(
-        model_id,
-        torch_dtype=dtype,
-        device_map=device,
-        trust_remote_code=trust_remote_code,
-    )
+
+    # Some vision models (DepthAnything, Segformer) aren't loadable via
+    # AutoModel.  Try progressively more specific Auto classes.
+    model = None
+    auto_classes = [
+        transformers.AutoModel,
+        transformers.AutoModelForImageClassification,
+    ]
+    if hasattr(transformers, "AutoModelForDepthEstimation"):
+        auto_classes.append(transformers.AutoModelForDepthEstimation)
+    if hasattr(transformers, "AutoModelForSemanticSegmentation"):
+        auto_classes.append(transformers.AutoModelForSemanticSegmentation)
+    if hasattr(transformers, "AutoModelForImageToImage"):
+        auto_classes.append(transformers.AutoModelForImageToImage)
+    for auto_cls in auto_classes:
+        try:
+            model = auto_cls.from_pretrained(
+                model_id,
+                torch_dtype=dtype,
+                device_map=device,
+                trust_remote_code=trust_remote_code,
+            )
+            break
+        except (ValueError, TypeError):
+            continue
+    if model is None:
+        raise ValueError(f"Could not load {model_id} with any AutoModel variant")
     model.eval()
 
     # Multi-modal models (CLIP, SigLIP) wrap a vision sub-model that
@@ -398,13 +419,27 @@ def torch_vision_forward(
     """Run a single forward pass on a HuggingFace vision model.
 
     Returns:
-        last_hidden_state as numpy array [batch, seq_len, hidden_size].
+        Feature tensor as numpy array.  Shape varies by model:
+        [B, seq_len, hidden] for ViT-like, [B, C, H, W] for CNN-like,
+        or [B, num_classes] for classification heads.
     """
     device = next(model.parameters()).device
     dtype = next(model.parameters()).dtype
     pv = torch.from_numpy(pixel_values).to(device=device, dtype=dtype)
     outputs = model(pixel_values=pv)
-    return outputs.last_hidden_state.cpu().numpy()
+    # Prefer last_hidden_state; fall back to logits or first tensor output.
+    if hasattr(outputs, "last_hidden_state") and outputs.last_hidden_state is not None:
+        return outputs.last_hidden_state.cpu().numpy()
+    if hasattr(outputs, "logits") and outputs.logits is not None:
+        return outputs.logits.cpu().numpy()
+    # Some models (DepthAnything) return predicted_depth or similar.
+    if hasattr(outputs, "predicted_depth") and outputs.predicted_depth is not None:
+        return outputs.predicted_depth.cpu().numpy()
+    # Generic fallback: first tensor in the output.
+    for v in outputs.values():
+        if hasattr(v, "cpu"):
+            return v.cpu().numpy()
+    raise ValueError(f"No usable tensor in model outputs: {type(outputs)}")
 
 
 # ---------------------------------------------------------------------------

testdata/cases/encoder/bros-base.yaml

@@ -7,6 +7,7 @@ inputs:
   prompts:
     - "The quick brown fox jumps over the lazy dog."
 
+skip_reason: "Requires bbox input not supported in test harness."
 level: "L4"
 
 notes: "BROS base. Document understanding with spatial layout."

testdata/cases/encoder/ernie-m-tiny.yaml

@@ -7,6 +7,7 @@ inputs:
   prompts:
     - "The quick brown fox jumps over the lazy dog."
 
+skip_reason: "Tokenizer incompatible with current tokenizers library."
 level: "L4"
 
 notes: "ERNIE-M tiny random. Cross-lingual encoder."

testdata/cases/encoder/layoutlmv2-base.yaml

@@ -7,6 +7,7 @@ inputs:
   prompts:
     - "The quick brown fox jumps over the lazy dog."
 
+skip_reason: "Requires detectron2 + bbox/image inputs not in test harness."
 level: "L4"
 
 notes: "LayoutLMv2 base. Multi-modal document understanding."

testdata/cases/encoder/mega-base.yaml

@@ -7,6 +7,7 @@ inputs:
   prompts:
     - "The quick brown fox jumps over the lazy dog."
 
+skip_reason: "Model type not recognized by current transformers version."
 level: "L4"
 
 notes: "MEGA base. Moving average equipped gated attention."

testdata/cases/encoder/nezha-cn-base.yaml

@@ -7,6 +7,7 @@ inputs:
   prompts:
     - "The quick brown fox jumps over the lazy dog."
 
+skip_reason: "Model type not recognized by current transformers version."
 level: "L4"
 
 notes: "NEZHA Chinese base. Relative position attention Chinese encoder."

testdata/cases/seq2seq/trocr-small.yaml

@@ -7,6 +7,7 @@ inputs:
   images:
     - "pipeline-cat-chonk.jpeg"
 
+skip_reason: "Vision-encoder-decoder requires pixel_values input; not a text seq2seq model."
 level: "L4"
 
 notes: "TrOCR small handwritten. OCR with vision encoder + text decoder."

testdata/cases/vision/layoutlmv3-base.yaml

@@ -7,6 +7,7 @@ inputs:
   images:
     - "pipeline-cat-chonk.jpeg"
 
+skip_reason: "Requires bbox + pixel_values + text inputs; not a simple vision model."
 level: "L4"
 
 notes: "LayoutLMv3 base. Unified text+image document model."

testdata/golden/encoder/clip-text-model.json

@@ -0,0 +1,48 @@
+{
+  "top1_id": 3,
+  "top2_id": 475,
+  "top10_ids": [
+    3,
+    475,
+    203,
+    340,
+    418,
+    63,
+    398,
+    230,
+    195,
+    444
+  ],
+  "top10_logits": [
+    "0x1.31d6c60000000p+2",
+    "0x1.5c7f040000000p+1",
+    "0x1.42eaf20000000p+1",
+    "0x1.3c99b80000000p+1",
+    "0x1.3343d40000000p+1",
+    "0x1.2449e60000000p+1",
+    "0x1.1a9a1a0000000p+1",
+    "0x1.0f66960000000p+1",
+    "0x1.0db5460000000p+1",
+    "0x1.0ce24a0000000p+1"
+  ],
+  "logits_summary": [
+    "0x1.31d6c60000000p+2",
+    "-0x1.cd320c0000000p+2",
+    "0x1.d0efe00810000p-4",
+    "0x1.06b8bf736e32bp+0"
+  ],
+  "input_ids": [
+    49406,
+    518,
+    3712,
+    2866,
+    3240,
+    18911,
+    962,
+    518,
+    10753,
+    1929,
+    269,
+    49407
+  ]
+}

testdata/golden/encoder/deberta-v3-base.json

@@ -0,0 +1,48 @@
+{
+  "top1_id": 428,
+  "top2_id": 696,
+  "top10_ids": [
+    428,
+    696,
+    377,
+    571,
+    628,
+    565,
+    105,
+    586,
+    716,
+    504
+  ],
+  "top10_logits": [
+    "0x1.3262c40000000p+2",
+    "0x1.30c2c00000000p+2",
+    "0x1.288ea00000000p+2",
+    "0x1.1d980a0000000p+2",
+    "0x1.1249a80000000p+2",
+    "0x1.02922a0000000p+2",
+    "0x1.ff88b60000000p+1",
+    "0x1.34f5880000000p+1",
+    "0x1.34100e0000000p+1",
+    "0x1.0c8e3e0000000p+1"
+  ],
+  "logits_summary": [
+    "0x1.3262c40000000p+2",
+    "-0x1.315c000000000p+2",
+    "-0x1.d0192441d5555p-11",
+    "0x1.66d2ffacbe659p-1"
+  ],
+  "input_ids": [
+    1,
+    279,
+    1538,
+    3258,
+    16123,
+    14929,
+    360,
+    262,
+    9118,
+    1560,
+    260,
+    2
+  ]
+}

testdata/golden/encoder/flaubert-base.json

@@ -0,0 +1,53 @@
+{
+  "top1_id": 70,
+  "top2_id": 644,
+  "top10_ids": [
+    70,
+    644,
+    538,
+    241,
+    212,
+    34,
+    625,
+    265,
+    426,
+    158
+  ],
+  "top10_logits": [
+    "0x1.fc43e80000000p+1",
+    "0x1.f018420000000p+1",
+    "0x1.e7c3820000000p+1",
+    "0x1.c9fe500000000p+1",
+    "0x1.a8cbc80000000p+1",
+    "0x1.a5f5a00000000p+1",
+    "0x1.72ed220000000p+1",
+    "0x1.70a8400000000p+1",
+    "0x1.5452080000000p+1",
+    "0x1.50fce00000000p+1"
+  ],
+  "logits_summary": [
+    "0x1.fc43e80000000p+1",
+    "-0x1.2545d00000000p+2",
+    "-0x1.510a94ab5f000p-3",
+    "0x1.28d022cfe3dcap+0"
+  ],
+  "input_ids": [
+    0,
+    1037,
+    3114,
+    2395,
+    3319,
+    15370,
+    10724,
+    742,
+    14037,
+    3914,
+    11186,
+    457,
+    2971,
+    9499,
+    41301,
+    16,
+    1
+  ]
+}