Load generation config from nested configs

src/transformers/generation/utils.py

@@ -407,17 +407,11 @@ def adjust_generation_fn(
                     **repo_loading_kwargs,
                 )
             except OSError:
+                # `self` already has a generation config created from model config so we can simply raise a warning
                 logger.info(
                     "Generation config file not found, using a generation config created from the model config."
                 )
-                self.generation_config = GenerationConfig.from_pretrained(
-                    pretrained_model_name_or_path,
-                    config_file_name="config.json",
-                    _from_auto=from_auto_class,
-                    _from_pipeline=from_pipeline,
-                    _from_model_config=True,
-                    **repo_loading_kwargs,
-                )
+
             # Load custom generate function if `pretrained_model_name_or_path` defines it (and override `generate`)
             if hasattr(self, "load_custom_generate") and trust_remote_code:
                 try:

tests/generation/test_utils.py

@@ -67,7 +67,6 @@
         AutoModelForImageTextToText,
         AutoModelForSeq2SeqLM,
         AutoModelForSpeechSeq2Seq,
-        AutoModelForVision2Seq,
         BartForConditionalGeneration,
         BartTokenizer,
         DataCollatorWithFlattening,
@@ -4401,7 +4400,7 @@ def test_generate_vision2text_conditioning(self):
         """Test that `decoder_input_ids` can be used to condition the generation in vision-to-text models"""
         pixel_values = floats_tensor((2, 3, 30, 30))
         conditioning_input = torch.tensor([[10], [10]])  # this should be the 2nd output token, after the BOS token
-        model = AutoModelForVision2Seq.from_pretrained(
+        model = AutoModelForImageTextToText.from_pretrained(
             "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-vit-gpt2"
         )
         pixel_values = pixel_values.to(torch_device)
@@ -4421,6 +4420,24 @@ def test_generate_vision2text_conditioning(self):
         self.assertTrue(np.array_equal(output_sequences_decoder_input_ids, output_sequences_input_ids))
         self.assertTrue(np.array_equal(output_sequences_decoder_input_ids[:, 1:2], conditioning_input))
 
+    @pytest.mark.generate
+    def test_load_generation_config_from_text_subconfig(self):
+        """
+        Tests that generation config is be loaded from model's `text_config` when not present
+        in the model repo. We should infer the text config correctly and re-use special tokens
+        for generation. See https://github.com/huggingface/transformers/issues/42794
+        """
+        model = AutoModelForImageTextToText.from_pretrained(
+            "hf-internal-testing/tiny-random-LlavaForConditionalGeneration-no-generation-config",
+            device_map=torch_device,
+        )
+        self.assertTrue(model.generation_config.eos_token_id is not None)
+        self.assertTrue(model.generation_config.bos_token_id is not None)
+        self.assertTrue(model.generation_config.pad_token_id is not None)
+
+        # test that we can generate without inputs, i.e. from BOS
+        _ = model.generate()
+
     @require_read_token
     @slow
     @require_torch_accelerator