[https://nvbugs/6655359][test] Shrink the Cosmos3-Nano T2V LPIPS gate to 9 frames, relax T2V/V2V thresholds, and unwaive

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json

@@ -26,6 +26,6 @@
   "torch_version": "2.12.0+cu130",
   "tensorrt_llm_version": "1.3.0rc25",
   "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
+  "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.",
   "sha256": "1fd9b0ab24de130f593056a32a7b8555fafbbf073b76c29a4633504870b1dad0"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json

@@ -25,6 +25,6 @@
   "torch_version": "2.12.0+cu130",
   "tensorrt_llm_version": "1.3.0rc25",
   "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
+  "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.",
   "sha256": "3f7c9b958807356ced2de1734e301dc837fa0b095f8fed1e29da764993926046"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json

@@ -25,6 +25,6 @@
   "torch_version": "2.12.0+cu130",
   "tensorrt_llm_version": "1.3.0rc25",
   "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
+  "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.",
   "sha256": "0ec80b5c906ae576deedf8fb48c55edd0c78203138608f12d0c439da11ab6f10"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json

@@ -34,6 +34,6 @@
   "torch_version": "2.12.0+cu130",
   "tensorrt_llm_version": "1.3.0rc25",
   "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
+  "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.",
   "sha256": "32c080983eb8d94d1da5d21378ea87912dadf018f00f15123a33f000640341ad"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json

@@ -24,6 +24,6 @@
   "torch_version": "2.12.0+cu130",
   "tensorrt_llm_version": "1.3.0rc25",
   "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
+  "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.",
   "sha256": "035f3e764e6a36159071178a2d7be6ec3cabc60899736099ff89a59c037e15e1"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json

@@ -6,24 +6,25 @@
   "negative_prompt": "",
   "height": 720,
   "width": 1280,
-  "num_frames": 189,
+  "num_frames": 9,
   "num_inference_steps": 35,
   "guidance_scale": 6.0,
   "seed": 42,
   "frame_rate": 24.0,
+  "note": "KPI config shrunk from the 189-frame 720P default (nvbugs/6655359): 9 frames = 3 latent frames = 2,760 video tokens, the smallest T2V shape exercising multi-latent-frame temporal attention; the default costs ~7 min of generation per CI run and adds no gate value.",
   "attention_backend": "VANILLA",
   "torch_compile": false,
   "deterministic_algorithms": false,
   "fp32_matmul_precision": "highest",
   "cudnn_allow_tf32": true,
   "lpips_net": "alex",
-  "lpips_threshold": 0.05,
+  "lpips_threshold": 0.2,
   "measured_lpips_at_creation": 0.0,
-  "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test",
+  "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): media is cut on the gating lane's own GPU (self-distance ~0), and the bar sits above the measured 9-frame cross-stepping floor (0.075, B300-cut media on B200) so a future CI GPU change does not force a re-cut; severe regressions (historical reds 0.45-0.61) still fire it.",
   "diffusers_version": "0.39.0",
   "torch_version": "2.12.0+cu130",
-  "tensorrt_llm_version": "1.3.0rc25",
-  "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
-  "sha256": "980849ba2f1ff1101c0dce2ac8897172212a614f23c3fb0cc6acbd970dd42976"
+  "tensorrt_llm_version": "1.3.0rc26",
+  "tensorrt_llm_commit": "5fb68830c919b8dbcc07cb1095f5e7d369509919",
+  "environment": "Native build, no container; NVIDIA B200 (sm100), umb-b200-236 - the GPU of the CI lane that gates this test (l0_b200.yml). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359); the threshold absorbs that.",
+  "sha256": "e70deaaaae9e5bcd929906b3af77476ac2a529a1cb02db1026b98952e7590d44"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json

@@ -20,13 +20,13 @@
   "fp32_matmul_precision": "highest",
   "cudnn_allow_tf32": true,
   "lpips_net": "alex",
-  "lpips_threshold": 0.05,
+  "lpips_threshold": 0.15,
   "measured_lpips_at_creation": 0.0,
-  "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test",
+  "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): the bar sits above the measured cross-stepping floor - this B300-cut media scores LPIPS 0.0749 on the B200 lane that gates it - so the gate catches severe model regressions (historical reds 0.45-0.61), not GPU/kernel drift, and survives a CI GPU change without re-cutting media. Self-regeneration on the cutting host remains 0.0.",
   "diffusers_version": "0.39.0",
   "torch_version": "2.12.0+cu130",
   "tensorrt_llm_version": "1.3.0rc25",
   "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95",
-  "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.",
+  "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). The trajectory is NOT bit-stable across GPU steppings: this media measures LPIPS 0.0749 on B200 (sm100), the lane that gates it (nvbugs/6655359); the threshold sits above that floor.",
   "sha256": "728c9bed1c25bf7de2b727949cc8c985f0f5bf03fd78e19847f4bcc7edeb450b"
 }

tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py

@@ -53,7 +53,13 @@
 COSMOS3_LPIPS_PROMPT = "A serene mountain landscape with snow-capped peaks and a flowing river"
 COSMOS3_LPIPS_HEIGHT = 720
 COSMOS3_LPIPS_WIDTH = 1280
-COSMOS3_LPIPS_T2V_NUM_FRAMES = 189
+# 9 frames = 3 latent frames (temporal VAE: floor((F-1)/4)+1), 2,760 video
+# tokens -- the smallest T2V shape that still exercises multi-latent-frame
+# temporal attention. The 720P default of 189 frames (44,160 tokens) costs ~7
+# minutes of generation per CI run and adds no gate value: a golden catches
+# severe regressions, and cross-stepping drift only grows with trajectory
+# length (0.020 at 1 frame -> 0.151 at 189).
+COSMOS3_LPIPS_T2V_NUM_FRAMES = 9
 COSMOS3_LPIPS_T2I_NUM_FRAMES = 1
 # 9 frames = 3 latent frames: latents (0, 1) are pinned to the V2V reference,
 # latent 2 (pixel frames 5-8) is generated. Frame 8 is the golden-compared frame.
@@ -64,6 +70,17 @@
 COSMOS3_LPIPS_SEED = 42
 COSMOS3_LPIPS_FRAME_RATE = 24.0
 COSMOS3_LPIPS_THRESHOLD = 0.05
+# T2V/V2V gate at a relaxed KPI-backstop band, not at 0.05: Cosmos3-Nano
+# trajectories are not bit-stable across GPU steppings (nvbugs/6655359 --
+# B300-cut media measured LPIPS 0.020/0.075/0.151 on B200 at 1/9/189 frames
+# with everything else held fixed), so a tight bar just re-fires whenever the
+# media host and the CI lane disagree. The bars sit above the 9-frame
+# cross-stepping floor (0.075) so the gates catch severe model regressions
+# (historical reds were 0.45-0.61) and survive a CI GPU change without
+# re-cutting media. The V2V golden is B300-cut and scores 0.070 on the B200
+# lane; the T2V golden is cut on B200, the lane's own GPU.
+COSMOS3_LPIPS_T2V_THRESHOLD = 0.20
+COSMOS3_LPIPS_V2V_THRESHOLD = 0.15
 COSMOS3_I2V_4STEP_MODEL_SUBPATH = "Cosmos3-Super-Image2Video-4Step"
 COSMOS3_I2V_4STEP_LPIPS_PROMPT = (
     "The orange sphere slowly rises while the camera pans right across the scene"
@@ -363,7 +380,7 @@ def test_cosmos3_nano_t2v_lpips_against_golden(_visual_gen_deps, tmp_path):
         golden_path,
         generated_path,
     )
-    _assert_lpips_below_threshold(score, COSMOS3_LPIPS_THRESHOLD)
+    _assert_lpips_below_threshold(score, COSMOS3_LPIPS_T2V_THRESHOLD)
 
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
@@ -383,7 +400,7 @@ def test_cosmos3_nano_v2v_lpips_against_golden(_visual_gen_deps, tmp_path):
         golden_path,
         generated_path,
     )
-    _assert_lpips_below_threshold(score, COSMOS3_LPIPS_THRESHOLD)
+    _assert_lpips_below_threshold(score, COSMOS3_LPIPS_V2V_THRESHOLD)
 
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")

tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py

@@ -411,7 +411,7 @@ def _cleanup_cuda():
 
 @contextlib.contextmanager
 def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]:
-    """Pin fp32-matmul arithmetic so LPIPS goldens are portable across hosts.
+    """Pin fp32-matmul arithmetic so LPIPS goldens are portable across torch stacks.
 
     NGC PyTorch containers default matmul TF32 on (``float32_matmul_precision
     == "high"``); PyPI torch defaults it off (``"highest"``). A model with fp32
@@ -420,9 +420,17 @@ def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]:
     ``transformer_cosmos3.py``) therefore produces a different trajectory under
     each default, and a golden cut under one fails under the other -- measured
     LPIPS-to-golden moved 0.132 -> 0.054 from this single flag. Pin "highest"
-    (IEEE fp32, measured bit-stable across torch 2.11/2.12 and B200/B300), and
-    pin cuDNN TF32 to its universal default so the second knob cannot drift.
-    bf16 compute -- all of the heavy kernels -- is unaffected by either knob.
+    (IEEE fp32, measured bit-stable across torch 2.11/2.12), and pin cuDNN TF32
+    to its universal default so the second knob cannot drift. bf16 compute --
+    all of the heavy kernels -- is unaffected by either knob.
+
+    The pin's contract stops at the torch stack: it does NOT make trajectories
+    bit-stable across GPU steppings. Kernel selection differs between sm100 and
+    sm103, and the divergence compounds along the denoising trajectory --
+    B300-cut Cosmos3-Nano media measured LPIPS 0.02 (1 frame) to 0.15 (189
+    frames) on B200 with everything else held fixed (nvbugs/6655359). Golden
+    thresholds must therefore sit above the measured cross-stepping floor of
+    their own trajectory, or the media must be cut on the gating lane's GPU.
 
     Applied per generation path rather than from
     ``_lpips_deterministic_algorithms``: that helper also wraps generation for

tests/integration/test_lists/test-db/l0_b200.yml

@@ -15,6 +15,15 @@ l0_b200:
       backend: pytorch
   tests:
   # ------------- PyTorch tests ---------------
+  # TEMPORARY (REMOVE BEFORE MERGE): these two tests are post_merge-gated, so a
+  # PR pipeline never schedules them and --extra-stage cannot pull them in (the
+  # gate is in the test-db terms). Listed here only to prove the relaxed
+  # thresholds pass on the B200 CI lane before merge; the post_merge block
+  # below remains their permanent home. Drop this block only AFTER reading the
+  # B200 stage verdict (nvbugs/6655359 exists because the last such proof-run
+  # result landed after the listing was already dropped and went unread).
+  - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (10)
+  - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden TIMEOUT (10)
   - unittest/others/test_kv_cache_transceiver.py::test_cpp_nixl_sync_transfer_stress
   - unittest/others/test_lora_manager.py
   - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
@@ -435,7 +444,7 @@ l0_b200:
   - examples/visual_gen/test_visual_gen_qwen_image.py::test_qwenimage_lpips_against_golden TIMEOUT (10)
   - examples/visual_gen/test_visual_gen_qwen_image.py::test_qwen_image_layered_lpips_against_golden TIMEOUT (10)
   - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2i_lpips_against_golden TIMEOUT (10)
-  - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (15)
+  - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (10)
   - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden TIMEOUT (10)
   - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_i2v_4step_lpips_against_golden TIMEOUT (20)
   - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_t2v_lpips_against_golden TIMEOUT (20)

tests/integration/test_lists/waives.txt

@@ -111,8 +111,6 @@ examples/test_deepseek_v4_pro.py::test_short_token_boundary_smoke SKIP (https://
 examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/6632606)
 examples/test_ray.py::test_ray_disaggregated_serving_python[tp2] SKIP (https://nvbugs/6601574)
 examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800)
-examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6655359)
-examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden SKIP (https://nvbugs/6655359)
 examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux1-nvfp4] SKIP (https://nvbugs/6572800)
 examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800)
 examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450)