From 1b028a65f242ae5ed8fae8d77afc14db0c4299ae Mon Sep 17 00:00:00 2001 From: Arnav66692 Date: Mon, 12 Oct 2026 17:28:22 -0700 Subject: [PATCH 01/10] fix: return per-image image features for HunYuan-VL batched generation --- .../models/hunyuan_vl/modeling_hunyuan_vl.py | 11 ++++- .../models/hunyuan_vl/modular_hunyuan_vl.py | 11 ++++- .../chameleon/test_modeling_chameleon.py | 4 ++ tests/models/gemma4/test_modeling_gemma4.py | 4 ++ .../test_modeling_gemma4_unified.py | 1 + .../hunyuan_vl/test_modeling_hunyuan_vl.py | 15 +++++++ tests/models/inkling/test_modeling_inkling.py | 4 ++ tests/test_modeling_common.py | 42 ++++++++++++------- 8 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py index 00d78217df68..28fdc4025ea0 100644 --- a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py +++ b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py @@ -979,7 +979,12 @@ def get_image_features( ) -> tuple | BaseModelOutputWithPooling: vision_dtype = next(self.vision_tower.parameters()).dtype pixel_values = pixel_values.to(vision_dtype) - return self.vision_tower(pixel_values, grid_thw=image_grid_thw, **kwargs) + image_outputs = self.vision_tower(pixel_values, grid_thw=image_grid_thw, **kwargs) + # merger: h*(w+1)+2 tokens per image (newline per row + begin/end); packed as (1, total, dim) + merged_h, merged_w = (image_grid_thw[:, 1:] // self.config.vision_config.spatial_merge_size).unbind(-1) + split_sizes = (merged_h * (merged_w + 1) + 2).tolist() + image_outputs.pooler_output = torch.split(image_outputs.pooler_output[0], split_sizes) + return image_outputs def get_placeholder_mask( self, @@ -1064,7 +1069,9 @@ def forward( mm_encoder_outputs["image"] = self.get_image_features(pixel_values, image_grid_thw, return_dict=True) if mm_encoder_outputs.get("image") is not None: - image_embeds = mm_encoder_outputs["image"].pooler_output.to(inputs_embeds.device, inputs_embeds.dtype) + image_embeds = torch.cat(mm_encoder_outputs["image"].pooler_output, dim=0).to( + inputs_embeds.device, inputs_embeds.dtype + ) image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, diff --git a/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py b/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py index 047fba93c4ca..dcb6d5892ae7 100644 --- a/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py +++ b/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py @@ -1224,7 +1224,12 @@ def get_image_features( ) -> tuple | BaseModelOutputWithPooling: vision_dtype = next(self.vision_tower.parameters()).dtype pixel_values = pixel_values.to(vision_dtype) - return self.vision_tower(pixel_values, grid_thw=image_grid_thw, **kwargs) + image_outputs = self.vision_tower(pixel_values, grid_thw=image_grid_thw, **kwargs) + # merger: h*(w+1)+2 tokens per image (newline per row + begin/end); packed as (1, total, dim) + merged_h, merged_w = (image_grid_thw[:, 1:] // self.config.vision_config.spatial_merge_size).unbind(-1) + split_sizes = (merged_h * (merged_w + 1) + 2).tolist() + image_outputs.pooler_output = torch.split(image_outputs.pooler_output[0], split_sizes) + return image_outputs def get_placeholder_mask( self, @@ -1281,7 +1286,9 @@ def forward( mm_encoder_outputs["image"] = self.get_image_features(pixel_values, image_grid_thw, return_dict=True) if mm_encoder_outputs.get("image") is not None: - image_embeds = mm_encoder_outputs["image"].pooler_output.to(inputs_embeds.device, inputs_embeds.dtype) + image_embeds = torch.cat(mm_encoder_outputs["image"].pooler_output, dim=0).to( + inputs_embeds.device, inputs_embeds.dtype + ) image_mask = self.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, diff --git a/tests/models/chameleon/test_modeling_chameleon.py b/tests/models/chameleon/test_modeling_chameleon.py index 2ca994b1569d..0e97eb674404 100644 --- a/tests/models/chameleon/test_modeling_chameleon.py +++ b/tests/models/chameleon/test_modeling_chameleon.py @@ -245,6 +245,10 @@ def test_batching_equivalence(self): def test_get_image_features_output(self, return_dict: bool | None): pass + @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") + def test_get_image_features_pooler_output(self): + pass + @unittest.skip("Skip get_image_features tests as those are tested via ChameleonVision2SeqModelTest instead") def test_get_image_features_hidden_states(self): pass diff --git a/tests/models/gemma4/test_modeling_gemma4.py b/tests/models/gemma4/test_modeling_gemma4.py index f0e1e812e451..80504e56c399 100644 --- a/tests/models/gemma4/test_modeling_gemma4.py +++ b/tests/models/gemma4/test_modeling_gemma4.py @@ -288,6 +288,10 @@ def test_get_image_features_attentions(self): def test_get_image_features_output(self, return_dict: bool | None): pass + @unittest.skip("The tester has no image in input dict") + def test_get_image_features_pooler_output(self): + pass + @unittest.skip("The tester has no videos in input dict") def test_get_video_features_hidden_states(self): pass diff --git a/tests/models/gemma4_unified/test_modeling_gemma4_unified.py b/tests/models/gemma4_unified/test_modeling_gemma4_unified.py index 78fdde7b0720..08bb08c7c624 100644 --- a/tests/models/gemma4_unified/test_modeling_gemma4_unified.py +++ b/tests/models/gemma4_unified/test_modeling_gemma4_unified.py @@ -244,6 +244,7 @@ def skip_mm_output_format(self): "test_get_image_features_hidden_states", "test_get_image_features_attentions", "test_get_image_features_output", + "test_get_image_features_pooler_output", "test_get_video_features_hidden_states", "test_get_video_features_attentions", "test_get_video_features_output", diff --git a/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py b/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py index c04bf685e568..6d236f63b99f 100644 --- a/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py +++ b/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py @@ -151,6 +151,21 @@ class HunYuanVLModelTest(VLMModelTest, unittest.TestCase): # HunYuanVL packs all images into one flat patch stream; pixel_values.shape[0] is total patches, not batch size. skip_test_image_features_output_shape = True + def test_generate_expanded_batch_keeps_each_prompt_image(self): + # Each beam must use its prompt's image, so first-step logits should match greedy decoding. + config, inputs = self.prepare_config_and_inputs_for_generate() + model = HunYuanVLForConditionalGeneration(config).to(torch_device).eval() + generation_kwargs = { + "max_new_tokens": 1, + "do_sample": False, + "return_dict_in_generate": True, + "output_logits": True, + } + with torch.no_grad(): + greedy = model.generate(**inputs, **generation_kwargs).logits[0] + beams = model.generate(**inputs, num_beams=2, num_return_sequences=2, **generation_kwargs).logits[0] + torch.testing.assert_close(beams, greedy.repeat_interleave(2, dim=0), rtol=1e-4, atol=1e-5) + def test_auto_model_uses_base_model(self): config = self.model_tester.get_config() model = AutoModel.from_config(config).to(self.model_tester.device) diff --git a/tests/models/inkling/test_modeling_inkling.py b/tests/models/inkling/test_modeling_inkling.py index a7b28f8cd59f..fa08aff6488b 100644 --- a/tests/models/inkling/test_modeling_inkling.py +++ b/tests/models/inkling/test_modeling_inkling.py @@ -176,6 +176,10 @@ def test_get_image_features_attentions(self): def test_get_image_features_output(self, return_dict: bool | None): pass + @unittest.skip("The tester has no image in input dict") + def test_get_image_features_pooler_output(self): + pass + @unittest.skip("The tester has no videos in input dict") def test_get_video_features_hidden_states(self): pass diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 80dcbd30bd60..d7adf50d5d89 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -5407,24 +5407,36 @@ def test_get_image_features_output(self, return_dict: bool | None): hidden_size, f"hidden_size mismatch, full shape: {last_hidden_state_shape}", ) - - if ( - model_class.__name__ in get_values(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES) - and outputs.pooler_output is not None - ): - self.assertEqual( - len(outputs.pooler_output), - self.model_tester.batch_size, - f"batch_size mismatch for `pooler_output`: {len(outputs.pooler_output)} != {self.model_tester.batch_size}", - ) - self.assertEqual( - outputs.pooler_output[0].ndim, - 2, - f"each sample in `pooler_output` should be a 2D array but got {outputs.pooler_output[0].ndim}", - ) else: self.assertIsInstance(outputs, tuple, "get_image_features() must return a tuple if return_dict=False") + def test_get_image_features_pooler_output(self): + # one 2D entry per image + for model_class in self.all_model_classes: + if model_class.__name__ not in get_values(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES) or not hasattr( + model_class, "get_image_features" + ): + continue + + config, inputs_dict = self._image_features_prepare_config_and_inputs() + model = model_class(config).eval().to(torch_device) + with torch.no_grad(): + pooler_output = model.get_image_features(**inputs_dict).pooler_output + if pooler_output is None: + continue + + num_images = self.model_tester.batch_size * getattr(self.model_tester, "num_images", 1) + self.assertEqual( + len(pooler_output), + num_images, + f"`pooler_output` should have one entry per image: {len(pooler_output)} != {num_images}", + ) + self.assertEqual( + pooler_output[0].ndim, + 2, + f"each sample in `pooler_output` should be a 2D array but got {pooler_output[0].ndim}", + ) + def test_get_image_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) From 6009d4a9ae182915be1e632148c9f049f0eb2375 Mon Sep 17 00:00:00 2001 From: Arnav66692 Date: Mon, 12 Oct 2026 22:44:49 -0700 Subject: [PATCH 02/10] test: clarify image feature checks and skip unrelated models --- src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py | 2 +- src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py | 2 +- tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py | 2 +- tests/test_modeling_common.py | 7 +++---- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py index 28fdc4025ea0..26d945b647ed 100644 --- a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py +++ b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py @@ -980,7 +980,7 @@ def get_image_features( vision_dtype = next(self.vision_tower.parameters()).dtype pixel_values = pixel_values.to(vision_dtype) image_outputs = self.vision_tower(pixel_values, grid_thw=image_grid_thw, **kwargs) - # merger: h*(w+1)+2 tokens per image (newline per row + begin/end); packed as (1, total, dim) + # Include row separators and image boundary tokens in each split. merged_h, merged_w = (image_grid_thw[:, 1:] // self.config.vision_config.spatial_merge_size).unbind(-1) split_sizes = (merged_h * (merged_w + 1) + 2).tolist() image_outputs.pooler_output = torch.split(image_outputs.pooler_output[0], split_sizes) diff --git a/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py b/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py index dcb6d5892ae7..b33c0f6d3f9c 100644 --- a/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py +++ b/src/transformers/models/hunyuan_vl/modular_hunyuan_vl.py @@ -1225,7 +1225,7 @@ def get_image_features( vision_dtype = next(self.vision_tower.parameters()).dtype pixel_values = pixel_values.to(vision_dtype) image_outputs = self.vision_tower(pixel_values, grid_thw=image_grid_thw, **kwargs) - # merger: h*(w+1)+2 tokens per image (newline per row + begin/end); packed as (1, total, dim) + # Include row separators and image boundary tokens in each split. merged_h, merged_w = (image_grid_thw[:, 1:] // self.config.vision_config.spatial_merge_size).unbind(-1) split_sizes = (merged_h * (merged_w + 1) + 2).tolist() image_outputs.pooler_output = torch.split(image_outputs.pooler_output[0], split_sizes) diff --git a/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py b/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py index 6d236f63b99f..ad74eea4e953 100644 --- a/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py +++ b/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py @@ -152,7 +152,7 @@ class HunYuanVLModelTest(VLMModelTest, unittest.TestCase): skip_test_image_features_output_shape = True def test_generate_expanded_batch_keeps_each_prompt_image(self): - # Each beam must use its prompt's image, so first-step logits should match greedy decoding. + # Compare first-step logits before beam search can diverge from greedy decoding. config, inputs = self.prepare_config_and_inputs_for_generate() model = HunYuanVLForConditionalGeneration(config).to(torch_device).eval() generation_kwargs = { diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index d7adf50d5d89..fd86f2025194 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -5411,11 +5411,10 @@ def test_get_image_features_output(self, return_dict: bool | None): self.assertIsInstance(outputs, tuple, "get_image_features() must return a tuple if return_dict=False") def test_get_image_features_pooler_output(self): - # one 2D entry per image for model_class in self.all_model_classes: - if model_class.__name__ not in get_values(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES) or not hasattr( - model_class, "get_image_features" - ): + if model_class.__name__ not in get_values(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES): + continue + if not hasattr(model_class, "get_image_features"): continue config, inputs_dict = self._image_features_prepare_config_and_inputs() From 2e4d2d16ccadf0608e414eaf53d8a6f8432c891b Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 13 Oct 2026 17:09:01 +0200 Subject: [PATCH 03/10] gotta fix the test, `num_images` isn't the actual batch size --- .../hunyuan_vl/test_modeling_hunyuan_vl.py | 15 ------- tests/test_modeling_common.py | 40 +++++++++++++++++-- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py b/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py index ad74eea4e953..c04bf685e568 100644 --- a/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py +++ b/tests/models/hunyuan_vl/test_modeling_hunyuan_vl.py @@ -151,21 +151,6 @@ class HunYuanVLModelTest(VLMModelTest, unittest.TestCase): # HunYuanVL packs all images into one flat patch stream; pixel_values.shape[0] is total patches, not batch size. skip_test_image_features_output_shape = True - def test_generate_expanded_batch_keeps_each_prompt_image(self): - # Compare first-step logits before beam search can diverge from greedy decoding. - config, inputs = self.prepare_config_and_inputs_for_generate() - model = HunYuanVLForConditionalGeneration(config).to(torch_device).eval() - generation_kwargs = { - "max_new_tokens": 1, - "do_sample": False, - "return_dict_in_generate": True, - "output_logits": True, - } - with torch.no_grad(): - greedy = model.generate(**inputs, **generation_kwargs).logits[0] - beams = model.generate(**inputs, num_beams=2, num_return_sequences=2, **generation_kwargs).logits[0] - torch.testing.assert_close(beams, greedy.repeat_interleave(2, dim=0), rtol=1e-4, atol=1e-5) - def test_auto_model_uses_base_model(self): config = self.model_tester.get_config() model = AutoModel.from_config(config).to(self.model_tester.device) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index fd86f2025194..5c65eade0373 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -5411,6 +5411,10 @@ def test_get_image_features_output(self, return_dict: bool | None): self.assertIsInstance(outputs, tuple, "get_image_features() must return a tuple if return_dict=False") def test_get_image_features_pooler_output(self): + """ + Generation assumes that encoded images always follow certain layout. This test is IMPORTANT + and should NEVER be skipped unless your model override beam-search expansion on `generate()` + """ for model_class in self.all_model_classes: if model_class.__name__ not in get_values(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES): continue @@ -5420,15 +5424,14 @@ def test_get_image_features_pooler_output(self): config, inputs_dict = self._image_features_prepare_config_and_inputs() model = model_class(config).eval().to(torch_device) with torch.no_grad(): - pooler_output = model.get_image_features(**inputs_dict).pooler_output + pooler_output = model.get_image_features(**inputs_dict, return_dict=True).pooler_output if pooler_output is None: continue - num_images = self.model_tester.batch_size * getattr(self.model_tester, "num_images", 1) self.assertEqual( len(pooler_output), - num_images, - f"`pooler_output` should have one entry per image: {len(pooler_output)} != {num_images}", + self.model_tester.batch_size, + f"`pooler_output` should have one entry per image: {len(pooler_output)} != {self.model_tester.batch_size}", ) self.assertEqual( pooler_output[0].ndim, @@ -5735,6 +5738,35 @@ def test_get_video_features_output(self, return_dict: bool | None): else: self.assertIsInstance(outputs, tuple, "get_video_features() must return a tuple if return_dict=False") + def test_get_video_features_pooler_output(self): + """ + Generation assumes that encoded images always follow certain layout. This test is IMPORTANT + and should NEVER be skipped unless your model override beam-search expansion on `generate()` + """ + for model_class in self.all_model_classes: + if model_class.__name__ not in get_values(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES): + continue + if not hasattr(model_class, "get_video_features"): + continue + + config, inputs_dict = self._video_features_prepare_config_and_inputs() + model = model_class(config).eval().to(torch_device) + with torch.no_grad(): + pooler_output = model.get_video_features(**inputs_dict, return_dict=True).pooler_output + if pooler_output is None: + continue + + self.assertEqual( + len(pooler_output), + self.model_tester.batch_size, + f"batch_size mismatch for `pooler_output`: {len(pooler_output)} != {self.model_tester.batch_size}", + ) + self.assertEqual( + pooler_output[0].ndim, + 2, + f"each sample in `pooler_output` should be a 2D array but got {pooler_output[0].ndim}", + ) + def test_get_video_features_hidden_states(self): def check_hidden_states_output(inputs_dict, config, model_class): model = model_class(copy.deepcopy(config)) From 297f46d705792fa023fae23888387ed46a7cf6f3 Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 13 Oct 2026 17:22:13 +0200 Subject: [PATCH 04/10] fix models except nemotron --- src/transformers/models/idefics2/modeling_idefics2.py | 4 +++- src/transformers/models/idefics3/modeling_idefics3.py | 1 + .../llava_next_video/modeling_llava_next_video.py | 2 +- src/transformers/models/smolvlm/modeling_smolvlm.py | 1 + src/transformers/models/smolvlm/modular_smolvlm.py | 1 + .../nemotron_h_omni/test_modeling_nemotron_h_omni.py | 10 ++++++++++ 6 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index c73814f645fd..7578192ab399 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -861,9 +861,11 @@ def get_image_features( image_hidden_states = image_outputs.last_hidden_state # Modality projection & resampling - image_outputs.pooler_output = self.connector( + image_features = self.connector( image_hidden_states, attention_mask=patch_attention_mask.view(pixel_values.size(0), -1) ) + image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) + image_outputs.pooler_output = image_features return image_outputs diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 3ef9f8917e16..6ea4721fd6af 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -607,6 +607,7 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) + image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) image_outputs.pooler_output = image_features return image_outputs diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 18c42b938bb7..b0158d15860c 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -640,7 +640,7 @@ def get_video_features( # Same as image features except that video has pooling layer video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) - video_outputs.pooler_output = torch.split(video_features, frames, dim=0) + video_outputs.pooler_output = torch.split(video_features.squeeze(1), frames, dim=0) return video_outputs diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 2172e97610bc..ed13c59860a8 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -560,6 +560,7 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) + image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) image_outputs.pooler_output = image_features return image_outputs diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 17c19fdf1c2a..5b963b8b3084 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -192,6 +192,7 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) + image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) image_outputs.pooler_output = image_features return image_outputs diff --git a/tests/models/nemotron_h_omni/test_modeling_nemotron_h_omni.py b/tests/models/nemotron_h_omni/test_modeling_nemotron_h_omni.py index fbce3a88e74c..98b9294bcc13 100644 --- a/tests/models/nemotron_h_omni/test_modeling_nemotron_h_omni.py +++ b/tests/models/nemotron_h_omni/test_modeling_nemotron_h_omni.py @@ -347,6 +347,11 @@ def test_get_image_features_output(self, return_dict: bool | None): def test_get_video_features_output(self, return_dict: bool | None): pass + @unittest.skip("FIXME raushan - should apply temporal factor while processing!") + # prob this model doesn't even work with beam/sampling atm, low prio + def test_get_video_features_pooler_output(self): + pass + def test_mismatching_num_image_tokens(self): # packed `pixel_values` have no batch dimension, so images are dropped or added by their patches config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -435,6 +440,11 @@ def test_get_image_features_output(self, return_dict: bool | None): def test_get_video_features_output(self, return_dict: bool | None): pass + @unittest.skip("FIXME raushan - should apply temporal factor while processing!") + # prob this model doesn't even work with beam/sampling atm, low prio + def test_get_video_features_pooler_output(self): + pass + @unittest.skip(reason="Audio tester isnt supported yet") def test_generate_from_multimodal_encoder_outputs_and_raw_data(self): pass From b599de9fad05c19910b6b9a4d2d30c85e0a919ae Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 13 Oct 2026 17:37:18 +0200 Subject: [PATCH 05/10] oops, meant this! --- .../models/llava_next_video/modeling_llava_next_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index b0158d15860c..dbe8fab9800e 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -640,7 +640,7 @@ def get_video_features( # Same as image features except that video has pooling layer video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) - video_outputs.pooler_output = torch.split(video_features.squeeze(1), frames, dim=0) + video_outputs.pooler_output = video_features.unflatten(0, (batch_size, frames)).flatten(1, 2) return video_outputs From cf36cf6e58d198eadd66efe8c744b22a38cbd002 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 15 Oct 2026 22:04:03 +0200 Subject: [PATCH 06/10] fix --- src/transformers/models/idefics2/modeling_idefics2.py | 6 ++++-- src/transformers/models/idefics3/modeling_idefics3.py | 6 ++++-- src/transformers/models/smolvlm/modeling_smolvlm.py | 6 ++++-- src/transformers/models/smolvlm/modular_smolvlm.py | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 7578192ab399..00f066ed2de2 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -836,6 +836,7 @@ def get_image_features( # Remove padding images - padding images are full 0. nb_values_per_image = pixel_values.shape[1:].numel() real_images_inds = (pixel_values == 0.0).sum(dim=(-1, -2, -3)) != nb_values_per_image + num_real_images_per_batch = real_images_inds.view(batch_size, num_images).sum(dim=1) pixel_values = pixel_values[real_images_inds].contiguous() # Handle the vision attention mask @@ -864,8 +865,7 @@ def get_image_features( image_features = self.connector( image_hidden_states, attention_mask=patch_attention_mask.view(pixel_values.size(0), -1) ) - image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) - image_outputs.pooler_output = image_features + image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] return image_outputs @@ -940,6 +940,8 @@ def forward( image_hidden_states = None if image_hidden_states is not None: + if isinstance(image_hidden_states, (list, tuple)): + image_hidden_states = torch.cat(image_hidden_states, dim=0) image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=inputs_embeds.device) # When we generate, we don't want to replace the potential image_token_id that we generated by images # that simply don't exist diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 6ea4721fd6af..9a7d521e0c0e 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -580,6 +580,7 @@ def get_image_features( # Remove padding images - padding images are full 0. nb_values_per_image = pixel_values.shape[1:].numel() real_images_inds = (pixel_values == 0.0).sum(dim=(-1, -2, -3)) != nb_values_per_image + num_real_images_per_batch = real_images_inds.view(batch_size, num_images).sum(dim=1) pixel_values = pixel_values[real_images_inds].contiguous() # Handle the vision attention mask @@ -607,8 +608,7 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) - image_outputs.pooler_output = image_features + image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] return image_outputs @@ -680,6 +680,8 @@ def forward( image_hidden_states = None if image_hidden_states is not None: + if isinstance(image_hidden_states, (list, tuple)): + image_hidden_states = torch.cat(image_hidden_states, dim=0) image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=inputs_embeds.device) # When we generate, we don't want to replace the potential image_token_id that we generated by images # that simply don't exist diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index ed13c59860a8..26fe64a3f728 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -534,6 +534,7 @@ def get_image_features( # If no images, leave one empty image. real_images_inds[0] |= ~torch.any(real_images_inds) + num_real_images_per_batch = real_images_inds.view(batch_size, num_images).sum(dim=1) pixel_values = pixel_values[real_images_inds].contiguous() # Handle the vision attention mask @@ -560,8 +561,7 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) - image_outputs.pooler_output = image_features + image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] return image_outputs @@ -625,6 +625,8 @@ def forward( image_hidden_states = None if image_hidden_states is not None: + if isinstance(image_hidden_states, (list, tuple)): + image_hidden_states = torch.cat(image_hidden_states, dim=0) image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=inputs_embeds.device) # When we generate, we don't want to replace the potential image_token_id that we generated by images # that simply don't exist diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 5b963b8b3084..83b17d94a766 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -166,6 +166,7 @@ def get_image_features( # If no images, leave one empty image. real_images_inds[0] |= ~torch.any(real_images_inds) + num_real_images_per_batch = real_images_inds.view(batch_size, num_images).sum(dim=1) pixel_values = pixel_values[real_images_inds].contiguous() # Handle the vision attention mask @@ -192,8 +193,7 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_features = image_features.unflatten(0, (batch_size, num_images)).flatten(1, 2) - image_outputs.pooler_output = image_features + image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] return image_outputs @@ -253,6 +253,8 @@ def forward( image_hidden_states = None if image_hidden_states is not None: + if isinstance(image_hidden_states, (list, tuple)): + image_hidden_states = torch.cat(image_hidden_states, dim=0) image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=inputs_embeds.device) # When we generate, we don't want to replace the potential image_token_id that we generated by images # that simply don't exist From cd19cf274da8208b408703937d3bbb785d746936 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 15 Oct 2026 22:42:44 +0200 Subject: [PATCH 07/10] oh no, flatten was wrong dim --- src/transformers/models/idefics2/modeling_idefics2.py | 4 +++- src/transformers/models/idefics3/modeling_idefics3.py | 4 +++- src/transformers/models/smolvlm/modeling_smolvlm.py | 7 +++++-- src/transformers/models/smolvlm/modular_smolvlm.py | 7 +++++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 00f066ed2de2..d1ff9f92d848 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -865,7 +865,9 @@ def get_image_features( image_features = self.connector( image_hidden_states, attention_mask=patch_attention_mask.view(pixel_values.size(0), -1) ) - image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] + image_outputs.pooler_output = [ + f.reshape(-1, f.shape[-1]) for f in image_features.split(num_real_images_per_batch.tolist()) + ] return image_outputs diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 9a7d521e0c0e..e66243911d3d 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -608,7 +608,9 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] + image_outputs.pooler_output = [ + f.reshape(-1, f.shape[-1]) for f in image_features.split(num_real_images_per_batch.tolist()) + ] return image_outputs diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index 26fe64a3f728..d30707851096 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -478,7 +478,8 @@ def inputs_merger( - The merging happens so that we obtain the following sequence: `vector_tok_1 vector_tok_2 vector_tok_3 vector_fake_tok_around_image {sequence of image_seq_len image hidden states} vector_fake_toke_around_image vector_tok_4`. That sequence is fed to the LM. - To fit the format of that sequence, `input_ids`, `inputs_embeds`, `attention_mask` are all 3 adapted to insert the image hidden states. """ - _, patch_size, _ = image_hidden_states.shape + patch_size = self.config.vision_config.patch_size + image_hidden_states = image_hidden_states.view(-1, patch_size, image_hidden_states.shape[-1]) if input_ids is None: image_mask = inputs_embeds == self.get_input_embeddings()( @@ -561,7 +562,9 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] + image_outputs.pooler_output = [ + f.reshape(-1, f.shape[-1]) for f in image_features.split(num_real_images_per_batch.tolist()) + ] return image_outputs diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 83b17d94a766..b6377cfd129b 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -110,7 +110,8 @@ class SmolVLMModel(Idefics3Model): def inputs_merger( self, input_ids: torch.LongTensor, inputs_embeds: torch.Tensor, image_hidden_states: torch.Tensor ): - _, patch_size, _ = image_hidden_states.shape + patch_size = self.config.vision_config.patch_size + image_hidden_states = image_hidden_states.view(-1, patch_size, image_hidden_states.shape[-1]) if input_ids is None: image_mask = inputs_embeds == self.get_input_embeddings()( @@ -193,7 +194,9 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_outputs.pooler_output = [f.flatten(-2) for f in image_features.split(num_real_images_per_batch.tolist())] + image_outputs.pooler_output = [ + f.reshape(-1, f.shape[-1]) for f in image_features.split(num_real_images_per_batch.tolist()) + ] return image_outputs From add6e34d693e423de70828b949358f58eed264fb Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 15 Oct 2026 22:46:51 +0200 Subject: [PATCH 08/10] fix repo as well --- .../models/llava_next_video/modular_llava_next_video.py | 2 +- .../models/modernvbert/modeling_modernvbert.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index afddc98dea4b..b272b3ca990c 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -339,7 +339,7 @@ def get_video_features( # Same as image features except that video has pooling layer video_features = self.vision_resampler(selected_video_features) video_features = self.multi_modal_projector(video_features) - video_outputs.pooler_output = torch.split(video_features, frames, dim=0) + video_outputs.pooler_output = video_features.unflatten(0, (batch_size, frames)).flatten(1, 2) return video_outputs def get_placeholder_mask( diff --git a/src/transformers/models/modernvbert/modeling_modernvbert.py b/src/transformers/models/modernvbert/modeling_modernvbert.py index ef329f6b7475..c09d87af3ad0 100755 --- a/src/transformers/models/modernvbert/modeling_modernvbert.py +++ b/src/transformers/models/modernvbert/modeling_modernvbert.py @@ -241,7 +241,8 @@ def inputs_merger( - The merging happens so that we obtain the following sequence: `vector_tok_1 vector_tok_2 vector_tok_3 vector_fake_tok_around_image {sequence of image_seq_len image hidden states} vector_fake_toke_around_image vector_tok_4`. That sequence is fed to the LM. - To fit the format of that sequence, `input_ids`, `inputs_embeds`, `attention_mask` are all 3 adapted to insert the image hidden states. """ - _, patch_size, _ = image_hidden_states.shape + patch_size = self.config.vision_config.patch_size + image_hidden_states = image_hidden_states.view(-1, patch_size, image_hidden_states.shape[-1]) if input_ids is None: image_mask = inputs_embeds == self.get_input_embeddings()( @@ -297,6 +298,7 @@ def get_image_features( # If no images, leave one empty image. real_images_inds[0] |= ~torch.any(real_images_inds) + num_real_images_per_batch = real_images_inds.view(batch_size, num_images).sum(dim=1) pixel_values = pixel_values[real_images_inds].contiguous() # Handle the vision attention mask @@ -323,7 +325,9 @@ def get_image_features( # Modality projection & resampling image_features = self.connector(image_hidden_states) - image_outputs.pooler_output = image_features + image_outputs.pooler_output = [ + f.reshape(-1, f.shape[-1]) for f in image_features.split(num_real_images_per_batch.tolist()) + ] return image_outputs From 2ed81fd28b633a79c01699a3c0b8361c1d482c14 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 15 Oct 2026 23:32:48 +0200 Subject: [PATCH 09/10] fix repo didnt copy it --- src/transformers/models/modernvbert/modeling_modernvbert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/modernvbert/modeling_modernvbert.py b/src/transformers/models/modernvbert/modeling_modernvbert.py index c09d87af3ad0..0f8bd63eccc8 100755 --- a/src/transformers/models/modernvbert/modeling_modernvbert.py +++ b/src/transformers/models/modernvbert/modeling_modernvbert.py @@ -373,6 +373,7 @@ def forward( # Merge image and text embeddings if image_hidden_states is not None: + image_hidden_states = torch.cat(image_hidden_states, dim=0) image_hidden_states = image_hidden_states.to(dtype=inputs_embeds.dtype, device=inputs_embeds.device) inputs_embeds = self.inputs_merger( input_ids=input_ids, inputs_embeds=inputs_embeds, image_hidden_states=image_hidden_states From 34447f7a97ed53cea7223d0f7b6ac29cfd1fb269 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 16 Oct 2026 02:25:10 +0200 Subject: [PATCH 10/10] i hare modular --- src/transformers/models/modernvbert/modular_modernvbert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/modernvbert/modular_modernvbert.py b/src/transformers/models/modernvbert/modular_modernvbert.py index ed1a8966d492..65329c1fe7f5 100755 --- a/src/transformers/models/modernvbert/modular_modernvbert.py +++ b/src/transformers/models/modernvbert/modular_modernvbert.py @@ -296,6 +296,7 @@ def forward( # Merge image and text embeddings if image_hidden_states is not None: + image_hidden_states = torch.cat(image_hidden_states, dim=0) image_hidden_states = image_hidden_states.to(dtype=inputs_embeds.dtype, device=inputs_embeds.device) inputs_embeds = self.inputs_merger( input_ids=input_ids, inputs_embeds=inputs_embeds, image_hidden_states=image_hidden_states