update vllm miltimodal for api calls convenience

.github/workflows/tests.yml

@@ -85,6 +85,7 @@ jobs:
           .
     - name: Run all tests
       env:
+        NV_INFERENCE_API_KEY: ${{ secrets.NV_INFERENCE_API_KEY }}
         NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |

.gitignore

@@ -49,3 +49,6 @@ AGENTS.md
 .codex
 
 .idea
+
+#scripts at root level
+/*.sh

nemo_skills/inference/generate.py

@@ -210,7 +210,7 @@ class GenerationTaskConfig:
     enable_litellm_cache: bool = False
 
     # List of content types to drop from messages (e.g., base64 audio) to keep output files smaller
-    drop_content_types: list[str] = field(default_factory=lambda: ["audio_url"])
+    drop_content_types: list[str] = field(default_factory=lambda: ["audio_url", "input_audio"])
 
     # Audio configuration - set by benchmarks that need audio processing (mmau-pro, audiobench, etc.)
     enable_audio: bool = False  # Enable audio preprocessing (set by benchmark configs)
@@ -632,7 +632,7 @@ def drop_fields_from_messages(self, output):
 
             # Filter out content types specified in drop_content_types config
             message["content"] = [
-                content for content in message["content"] if content.get("type") not in self.cfg.drop_content_types
+                content for content in message["content"] if content["type"] not in self.cfg.drop_content_types
             ]
 
     async def postprocess_single_output(self, output, original_data_point):

nemo_skills/inference/model/audio_utils.py

@@ -145,6 +145,8 @@ def make_audio_content_block(base64_audio: str, audio_format: str = "audio_url")
     if audio_format == "input_audio":
         # OpenAI native format (works with NVIDIA API / Gemini / Azure)
         return {"type": "input_audio", "input_audio": {"data": base64_audio, "format": "wav"}}
-    else:
+    elif audio_format == "audio_url":
         # Data URI format (works with vLLM / Qwen)
         return {"type": "audio_url", "audio_url": {"url": f"data:audio/wav;base64,{base64_audio}"}}
+    else:
+        raise ValueError(f"Unsupported audio_format '{audio_format}'. Use 'audio_url' or 'input_audio'.")