diff --git a/scripts/visualgen_eval/visual_gen_lpips_score_eval.py b/scripts/visualgen_eval/visual_gen_lpips_score_eval.py index 9697655f72ea..bb7d926a4daa 100644 --- a/scripts/visualgen_eval/visual_gen_lpips_score_eval.py +++ b/scripts/visualgen_eval/visual_gen_lpips_score_eval.py @@ -762,6 +762,14 @@ def _evaluate(args: argparse.Namespace) -> dict[str, Any]: def main() -> None: + # Pin fp32-matmul arithmetic for scoring, mirroring the generation-side pin + # (_lpips_pinned_fp32_matmul_precision in visual_gen_test_utils.py). This + # script runs as a subprocess, so the parent's torch flags do not propagate; + # without the pin the scorer inherits the host default, which differs + # between NGC containers (TF32 on) and PyPI torch (TF32 off). + torch.set_float32_matmul_precision("highest") + torch.backends.cudnn.allow_tf32 = True + args = parse_args() result = _evaluate(args) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json index 00f4e3ececa9..596e180410bb 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json @@ -1,27 +1,31 @@ { - "source": "diffusers Cosmos3OmniPipeline on diffusers main (reference implementation, not a TRT-LLM self-golden)", - "diffusers_reference": "huggingface/diffusers#14181 'Cosmos3 edge support' + #14246 'Fix Cosmos3 Edge generator K normalization'", - "diffusers_version": "0.40.0.dev0", - "diffusers_commit": "2919c50968389232c527bdab1a3af69cef01ed07", - "scheduler_override": "UniPCMultistepScheduler.from_config(checkpoint config, use_karras_sigmas=False, flow_shift=3.0). With the checkpoint's use_native_flow_schedule=true this reproduces the cosmos-framework PyTorch backend schedule (fm_solvers_unipc @ 117c7d2) to fp32-ulp: timesteps bit-identical, full synthetic step() trajectories agree to <=1.6e-7 rel (see TestNativeFlowSchedule fixtures). Stock diffusers is NOT used as-is because its karras branch swallows the native flow sigmas.", - "prompt_text_matching": "The golden run passed pre-formatted cond AND uncond texts with add_duration_template=False and add_resolution_template=False. Both texts were produced by TRT-LLM's _format_prompt_with_metadata (keep-metadata negative-prompt semantics, matching cosmos-framework's CLI default rather than diffusers' inverse templates), so both stacks tokenize identical sequences in both CFG branches.", - "model": "Cosmos3-Edge", - "seed": 42, - "generator": "torch.Generator(device='cuda').manual_seed(42); initial latents match TRT-LLM's randn_tensor draw bit-for-bit (same shape/dtype/generator semantics)", - "use_system_prompt": false, - "torch_dtype": "bfloat16", - "lpips_net": "alex", "video": "cosmos3_edge_i2v_lpips_golden_video.mp4", + "model": "Cosmos3-Edge", + "source": "TensorRT-LLM VisualGen (self-golden)", "prompt": "The orange sphere slowly rises while the camera pans right across the scene", - "conditioning_image": "deterministic 832x480 image drawn by _write_cosmos3_edge_conditioning_image in test_visual_gen.py", + "negative_prompt": "", "height": 480, "width": 832, "num_frames": 29, "num_inference_steps": 10, "guidance_scale": 5.0, + "seed": 42, "frame_rate": 24.0, + "conditioning_image": "deterministic 832x480 image drawn by _write_cosmos3_edge_conditioning_image in test_visual_gen_cosmos3.py", + "note": "Regression gate only (TRT-LLM vs itself under pinned arithmetic). Cross-stack correctness vs diffusers is covered by the per-step parity unit test (test_cosmos3_edge.py::TestDiffusersParity).", + "attention_backend": "VANILLA", + "torch_compile": false, + "deterministic_algorithms": false, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, + "lpips_net": "alex", "lpips_threshold": 0.13, - "measured_lpips_at_creation": 0.0778, - "threshold_rationale": "0.0778 measured cross-stack at 10 steps (I2V accumulates cross-stack drift faster than T2V: 0.1105 at the deployed 50 steps), plus ~0.04 cross-host headroom. The failure signal is far away: a wrong-seed run against this golden measures LPIPS 0.858. The deployed 50-step I2V shape is exercised by test_cosmos3_edge_i2v_example.", - "notes": "Per-step masked-velocity parity vs diffusers is 0.8-1.5 percent rel (noisy frames); diffusers zeroes the conditioned frame's velocity while TRT-LLM masks it in the pipeline - equivalent for the scheduler." + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "diffusers_version": "0.39.0", + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "1fd9b0ab24de130f593056a32a7b8555fafbbf073b76c29a4633504870b1dad0" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json index 0a5331b8bbc1..92ad6e4ba1f3 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json @@ -1,24 +1,30 @@ { - "source": "diffusers Cosmos3OmniPipeline on diffusers main (reference implementation, not a TRT-LLM self-golden)", - "diffusers_reference": "huggingface/diffusers#14181 'Cosmos3 edge support' + #14246 'Fix Cosmos3 Edge generator K normalization'", - "diffusers_version": "0.40.0.dev0", - "diffusers_commit": "2919c50968389232c527bdab1a3af69cef01ed07", - "scheduler_override": "UniPCMultistepScheduler.from_config(checkpoint config, use_karras_sigmas=False, flow_shift=3.0). With the checkpoint's use_native_flow_schedule=true this reproduces the cosmos-framework PyTorch backend schedule (fm_solvers_unipc @ 117c7d2) to fp32-ulp: timesteps bit-identical, full synthetic step() trajectories agree to <=1.6e-7 rel (see TestNativeFlowSchedule fixtures). Stock diffusers is NOT used as-is because its karras branch swallows the native flow sigmas.", - "prompt_text_matching": "The golden run passed pre-formatted cond AND uncond texts with add_duration_template=False and add_resolution_template=False. Both texts were produced by TRT-LLM's _format_prompt_with_metadata (keep-metadata negative-prompt semantics, matching cosmos-framework's CLI default rather than diffusers' inverse templates), so both stacks tokenize identical sequences in both CFG branches.", - "model": "Cosmos3-Edge", - "seed": 42, - "generator": "torch.Generator(device='cuda').manual_seed(42); initial latents match TRT-LLM's randn_tensor draw bit-for-bit (same shape/dtype/generator semantics)", - "use_system_prompt": false, - "torch_dtype": "bfloat16", - "lpips_net": "alex", "image": "cosmos3_edge_t2i_lpips_golden.png", + "model": "Cosmos3-Edge", + "source": "TensorRT-LLM VisualGen (self-golden)", "prompt": "A ceramic teapot pouring steaming tea into a cup, morning window light", + "negative_prompt": "", "height": 640, "width": 640, "num_frames": 1, "num_inference_steps": 50, "guidance_scale": 4.0, + "seed": 42, + "output_type": "image", + "note": "Regression gate only (TRT-LLM vs itself under pinned arithmetic). Cross-stack correctness vs diffusers is covered by the per-step parity unit test (test_cosmos3_edge.py::TestDiffusersParity).", + "attention_backend": "VANILLA", + "torch_compile": false, + "deterministic_algorithms": false, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, + "lpips_net": "alex", "lpips_threshold": 0.05, - "measured_lpips_at_creation": 0.0056, - "threshold_rationale": "0.0056 measured cross-stack on B200; 0.05 matches the FLUX/QwenImage image-gate convention." + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "diffusers_version": "0.39.0", + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "3f7c9b958807356ced2de1734e301dc837fa0b095f8fed1e29da764993926046" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json index 904374e075ef..3b053b41ad68 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json @@ -1,25 +1,30 @@ { - "source": "diffusers Cosmos3OmniPipeline on diffusers main (reference implementation, not a TRT-LLM self-golden)", - "diffusers_reference": "huggingface/diffusers#14181 'Cosmos3 edge support' + #14246 'Fix Cosmos3 Edge generator K normalization'", - "diffusers_version": "0.40.0.dev0", - "diffusers_commit": "2919c50968389232c527bdab1a3af69cef01ed07", - "scheduler_override": "UniPCMultistepScheduler.from_config(checkpoint config, use_karras_sigmas=False, flow_shift=3.0). With the checkpoint's use_native_flow_schedule=true this reproduces the cosmos-framework PyTorch backend schedule (fm_solvers_unipc @ 117c7d2) to fp32-ulp: timesteps bit-identical, full synthetic step() trajectories agree to <=1.6e-7 rel (see TestNativeFlowSchedule fixtures). Stock diffusers is NOT used as-is because its karras branch swallows the native flow sigmas.", - "prompt_text_matching": "The golden run passed pre-formatted cond AND uncond texts with add_duration_template=False and add_resolution_template=False. Both texts were produced by TRT-LLM's _format_prompt_with_metadata (keep-metadata negative-prompt semantics, matching cosmos-framework's CLI default rather than diffusers' inverse templates), so both stacks tokenize identical sequences in both CFG branches.", - "model": "Cosmos3-Edge", - "seed": 42, - "generator": "torch.Generator(device='cuda').manual_seed(42); initial latents match TRT-LLM's randn_tensor draw bit-for-bit (same shape/dtype/generator semantics)", - "use_system_prompt": false, - "torch_dtype": "bfloat16", - "lpips_net": "alex", "video": "cosmos3_edge_t2v_lpips_golden_video.mp4", + "model": "Cosmos3-Edge", + "source": "TensorRT-LLM VisualGen (self-golden)", "prompt": "A red ball rolls across a wooden floor, casting a soft shadow.", + "negative_prompt": "", "height": 480, "width": 832, "num_frames": 29, "num_inference_steps": 50, "guidance_scale": 5.0, + "seed": 42, "frame_rate": 24.0, + "note": "Regression gate only (TRT-LLM vs itself under pinned arithmetic). Cross-stack correctness vs diffusers is covered by the per-step parity unit test (test_cosmos3_edge.py::TestDiffusersParity).", + "attention_backend": "VANILLA", + "torch_compile": false, + "deterministic_algorithms": false, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, + "lpips_net": "alex", "lpips_threshold": 0.1, - "measured_lpips_at_creation": 0.0447, - "threshold_rationale": "0.0447 measured cross-stack (TRT-LLM VANILLA attention vs diffusers main) on B200 with matched noise and matched CFG texts, plus headroom for the ~0.04 cross-host kernel drift documented in _preserve_lpips_candidate_on_failure." + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "diffusers_version": "0.39.0", + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "0ec80b5c906ae576deedf8fb48c55edd0c78203138608f12d0c439da11ab6f10" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json index 0abf4430cc0c..55d4cf08a551 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json @@ -1,27 +1,39 @@ { "image": "cosmos3_nano_fp8_blockwise_lpips_golden.png", "model": "Cosmos3-Nano", - "source": "TensorRT-LLM VisualGen", + "source": "TensorRT-LLM VisualGen (self-golden)", "prompt": "A serene mountain landscape with snow-capped peaks and a flowing river", + "negative_prompt": "", "height": 720, "width": 1280, "num_frames": 1, "num_inference_steps": 35, "guidance_scale": 6.0, - "frame_rate": 24.0, "seed": 42, + "frame_rate": 24.0, "feature_config": { "quantization": "FP8_BLOCK_SCALES", "cuda_graph": false }, + "quantization_ignore": [ + "language_model.*", + "vae2llm", + "llm2vae", + "time_embedder.*" + ], + "attention_backend": "VANILLA", "torch_compile": false, "deterministic_algorithms": true, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, "lpips_net": "alex", "lpips_threshold": 0.05, + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", - "torch_version": "2.12.0a0+5aff3928d8.nv26.05", - "tensorrt_llm_version": "1.3.0rc21", - "tensorrt_llm_commit": "b2131b181f5be6717cd302a0b53c22c6a70c65b3", - "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm@sha256:475999862b896593159f10f486e16a748e7b7abb5cee558932a59ea6fc690d6b", - "sha256": "dbdfc83e3a6a038138f4c8e039fa09be8ab11cb308f1534405624523afed73a5" + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "32c080983eb8d94d1da5d21378ea87912dadf018f00f15123a33f000640341ad" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json index f0bfb8639013..c09d9cc280a0 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json @@ -1,21 +1,29 @@ { "image": "cosmos3_nano_t2i_lpips_golden.png", "model": "Cosmos3-Nano", - "source": "TensorRT-LLM VisualGen", + "source": "TensorRT-LLM VisualGen (self-golden)", "prompt": "A serene mountain landscape with snow-capped peaks and a flowing river", + "negative_prompt": "", "height": 720, "width": 1280, "num_frames": 1, "num_inference_steps": 35, "guidance_scale": 6.0, "seed": 42, + "frame_rate": 24.0, "attention_backend": "VANILLA", "torch_compile": false, - "deterministic_algorithms": true, + "deterministic_algorithms": false, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, "lpips_net": "alex", "lpips_threshold": 0.05, - "diffusers_version": "0.38.0", - "tensorrt_llm_version": "1.3.0rc20", - "tensorrt_llm_commit": "85665f5fd331d0154a78172954846d843085e83f", - "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:3308a2dc0192a8329ea02eca7b5c44f290f5e894cd8c5921099308d84c3e5691" + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "diffusers_version": "0.39.0", + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "035f3e764e6a36159071178a2d7be6ec3cabc60899736099ff89a59c037e15e1" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json index 450d085268df..ee6014644db1 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json @@ -1,8 +1,9 @@ { "video": "cosmos3_nano_t2v_lpips_golden_video.mp4", "model": "Cosmos3-Nano", - "source": "TensorRT-LLM VisualGen", + "source": "TensorRT-LLM VisualGen (self-golden)", "prompt": "A serene mountain landscape with snow-capped peaks and a flowing river", + "negative_prompt": "", "height": 720, "width": 1280, "num_frames": 189, @@ -12,11 +13,17 @@ "frame_rate": 24.0, "attention_backend": "VANILLA", "torch_compile": false, - "deterministic_algorithms": true, + "deterministic_algorithms": false, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, "lpips_net": "alex", "lpips_threshold": 0.05, - "diffusers_version": "0.38.0", - "tensorrt_llm_version": "1.3.0rc20", - "tensorrt_llm_commit": "85665f5fd331d0154a78172954846d843085e83f", - "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:3308a2dc0192a8329ea02eca7b5c44f290f5e894cd8c5921099308d84c3e5691" + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "diffusers_version": "0.39.0", + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "980849ba2f1ff1101c0dce2ac8897172212a614f23c3fb0cc6acbd970dd42976" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json new file mode 100644 index 000000000000..636b8c3b484f --- /dev/null +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json @@ -0,0 +1,32 @@ +{ + "image": "cosmos3_nano_v2v_lpips_golden_frame.png", + "model": "Cosmos3-Nano", + "source": "TensorRT-LLM VisualGen (self-golden)", + "prompt": "A serene mountain landscape with snow-capped peaks and a flowing river", + "negative_prompt": "", + "height": 720, + "width": 1280, + "num_frames": 9, + "num_inference_steps": 35, + "guidance_scale": 6.0, + "seed": 42, + "frame_rate": 24.0, + "free_frame_index": 8, + "conditioning_video": "tests/integration/defs/examples/visual_gen/test_data/cosmos3_v2v_lpips_reference.mp4 (in-repo fixture; H.264 decode is bit-exact by spec, NVDEC output deterministic)", + "note": "First provenance record for this golden; the original (#16155, 2026-08-03) shipped without one.", + "attention_backend": "VANILLA", + "torch_compile": false, + "deterministic_algorithms": false, + "fp32_matmul_precision": "highest", + "cudnn_allow_tf32": true, + "lpips_net": "alex", + "lpips_threshold": 0.05, + "measured_lpips_at_creation": 0.0, + "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "diffusers_version": "0.39.0", + "torch_version": "2.12.0+cu130", + "tensorrt_llm_version": "1.3.0rc25", + "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", + "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "sha256": "728c9bed1c25bf7de2b727949cc8c985f0f5bf03fd78e19847f4bcc7edeb450b" +} diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip index 8b553103413c..c298c535a41b 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7043356a539371abc61e23c275df3e8388ff4d80894bf9d7fb7845eaf764741b -size 38528317 +oid sha256:33da5841fe5594a739240d048a0c6139c58e7e8e04d76cce1fb0cce968ead133 +size 30674088 diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py index bb72a4656f9f..290028266292 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py @@ -15,6 +15,7 @@ """Single-GPU integration and accuracy tests for Cosmos3.""" +import contextlib import os from dataclasses import dataclass @@ -35,6 +36,7 @@ _golden_media_path, _lpips_deterministic_algorithms, _lpips_model_path, + _lpips_pinned_fp32_matmul_precision, _preserve_lpips_candidate_on_failure, _run_lpips_eval, _run_reusable_image_lpips_eval, @@ -160,7 +162,9 @@ def _run_cosmos3_lpips_pipeline(num_frames, video=None): ) pipeline = PipelineLoader(args).load(skip_warmup=True) try: - with torch.no_grad(): + # Pin fp32-matmul arithmetic: the goldens are cut and compared + # under "highest" so they reproduce on both PyPI and NGC torch. + with torch.no_grad(), _lpips_pinned_fp32_matmul_precision(): result = pipeline.forward( prompt=COSMOS3_LPIPS_PROMPT, # The goldens were generated against an empty uncond branch, @@ -246,7 +250,22 @@ def _generate_cosmos3_feature_image(case, output_path): model_path = _lpips_model_path(COSMOS3_NANO_MODEL_SUBPATH) _skip_if_missing(model_path, "Cosmos3-Nano checkpoint", is_dir=True) _disable_inductor_compile_worker_quiesce() - with _lpips_deterministic_algorithms(), _fixed_nvfp4_quantization_backend(case.features): + # Pin fp32-matmul arithmetic only for the profiles whose goldens are + # re-baselined under it. NVFP4's golden is waived (nvbugs/6572800) and + # not re-cut here, so it keeps generating under the host default; pin + # it when that golden is re-cut. Note the NVFP4 profile reaches this + # same function inside a spawned process, so the guard has to be here + # rather than at the call site. + precision_context = ( + contextlib.nullcontext() + if case.features.quantization == "NVFP4" + else _lpips_pinned_fp32_matmul_precision() + ) + with ( + _lpips_deterministic_algorithms(), + precision_context, + _fixed_nvfp4_quantization_backend(case.features), + ): args = _build_single_device_feature_args( model_path, case.features, @@ -569,6 +588,12 @@ def _run_cosmos3_i2v_4step_lpips_pipeline(image_path): ) pipeline = PipelineLoader(args).load(skip_warmup=True) try: + # Deliberately NOT pinned with _lpips_pinned_fp32_matmul_precision: + # this golden is a diffusers cross-stack reference whose provenance + # records no fp32-matmul state, so whether pinning improves or + # degrades agreement is unknown. The test is skipped in CI anyway + # (checkpoint absent), and its golden is not re-cut here. Pin this + # path when the golden is re-cut and the flag can be recorded. with torch.no_grad(): result = pipeline.forward( prompt=COSMOS3_I2V_4STEP_LPIPS_PROMPT, @@ -760,7 +785,7 @@ def _run_cosmos3_edge_lpips_pipeline(**forward_kwargs): # The goldens were generated against an empty uncond branch, so pin it # here rather than inheriting the video-mode default negative prompt. forward_kwargs.setdefault("negative_prompt", "") - with torch.no_grad(): + with torch.no_grad(), _lpips_pinned_fp32_matmul_precision(): result = pipeline.forward( seed=COSMOS3_EDGE_LPIPS_SEED, use_guardrails=False, diff --git a/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py b/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py index 714d49652d88..14bc3c16a24d 100644 --- a/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py +++ b/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py @@ -26,7 +26,7 @@ import traceback import zipfile from dataclasses import dataclass -from typing import Collection, Literal +from typing import Collection, Iterator, Literal import pytest import torch @@ -409,6 +409,38 @@ def _cleanup_cuda(): shutdown_compile_workers() +@contextlib.contextmanager +def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]: + """Pin fp32-matmul arithmetic so LPIPS goldens are portable across hosts. + + NGC PyTorch containers default matmul TF32 on (``float32_matmul_precision + == "high"``); PyPI torch defaults it off (``"highest"``). A model with fp32 + GEMMs inside its denoising loop (Cosmos3: the RoPE frequency matmul, the + fp32 timestep embedder, and the fp32 autocast block in + ``transformer_cosmos3.py``) therefore produces a different trajectory under + each default, and a golden cut under one fails under the other -- measured + LPIPS-to-golden moved 0.132 -> 0.054 from this single flag. Pin "highest" + (IEEE fp32, measured bit-stable across torch 2.11/2.12 and B200/B300), and + pin cuDNN TF32 to its universal default so the second knob cannot drift. + bf16 compute -- all of the heavy kernels -- is unaffected by either knob. + + Applied per generation path rather than from + ``_lpips_deterministic_algorithms``: that helper also wraps generation for + LTX-2, HunyuanVideo, FLUX, QwenImage and WAN, whose goldens were cut + without the pin and are not re-baselined here. Keep the blast radius equal + to the goldens a change actually re-cuts. + """ + previous_precision = torch.get_float32_matmul_precision() + previous_cudnn_tf32 = torch.backends.cudnn.allow_tf32 + try: + torch.set_float32_matmul_precision("highest") + torch.backends.cudnn.allow_tf32 = True + yield + finally: + torch.set_float32_matmul_precision(previous_precision) + torch.backends.cudnn.allow_tf32 = previous_cudnn_tf32 + + @contextlib.contextmanager def _lpips_deterministic_algorithms(*, fully_eager=False): previous_deterministic = torch.are_deterministic_algorithms_enabled() diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d664cbfd06d9..913d0848e6bc 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -113,8 +113,6 @@ examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_accep examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights SKIP (https://nvbugs/6630699) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/6632606) examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800) -examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2i_lpips_against_golden SKIP (https://nvbugs/6418815) -examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6437341) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux1-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450)