From 4d1d0db9e51f99978a9fa44f5672c7c8f7b81415 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Mon, 23 Feb 2026 15:47:22 +0200
Subject: [PATCH 01/39] add support for transformers parakeet audio encoder for
 nemotron-nano-v3 avlm

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py |  2 +
 vllm/model_executor/models/parakeet.py        | 57 +++++++++++++++++--
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 51b36b1cae38..66ae76beb8e2 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1355,6 +1355,8 @@ def get_replacement_custom(item_idx: int):
                     image_height=image_size.height,
                     max_num_tiles=max_num_tiles,
                 )
+            msg = f"Determined num tokens of image: {feature_size=}"
+            logger.debug(msg)
 
             num_patches = None
             local_image_num_patches = image_num_patches
diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index 8c5539251c45..c86916b0d5ff 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -1,21 +1,44 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Modules below used for the audio encoder component in: models/nano_nemotron_vl.py
-"""
+
 
 from collections.abc import Iterable
-from dataclasses import asdict
+from dataclasses import asdict, dataclass
 
 import numpy as np
 import torch
 import torch.nn as nn
 from transformers import ParakeetEncoder as HFParakeetEncoder
-from transformers import ParakeetFeatureExtractor, PretrainedConfig
+from transformers import (
+    ParakeetEncoderConfig,
+    ParakeetFeatureExtractor,
+    PretrainedConfig,
+)
 
 from vllm.model_executor.layers.activation import ReLUSquaredActivation
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.transformers_utils.configs.parakeet import ExtractorConfig, ParakeetConfig
+
+
+class ParakeetConfig(ParakeetEncoderConfig):
+    llm_hidden_size: int
+    projection_hidden_size: int
+    projection_bias: bool
+    projection_eps: float = 1e-5
+    sampling_rate: int
+
+    @staticmethod
+    def from_hf_config(
+        config: PretrainedConfig, *, llm_hidden_size: int, max_model_len: int
+    ) -> "ParakeetConfig":
+        assert isinstance(config, PretrainedConfig)
+        return ParakeetConfig(
+            **config.to_dict(),
+            scale_input=False,
+            attention_bias=False,
+            llm_hidden_size=llm_hidden_size,
+            max_position_embeddings=max_model_len
+            + 1,  # + 1 because it seems like max_model_len+1 can be passed
+        )
 
 
 class ParakeetProjection(nn.Module):
@@ -102,6 +125,28 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         return loaded_params
 
 
+@dataclass(kw_only=True, frozen=True)
+class ExtractorConfig:
+    feature_size: int
+    sampling_rate: int
+    subsampling_factor: int
+    subsampling_conv_kernel_size: int
+    subsampling_conv_stride: int
+    clip_duration_s: int = 30
+    clip_min_duration_s: float = 0.1
+
+    @staticmethod
+    def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
+        assert isinstance(config, PretrainedConfig)
+        return ExtractorConfig(
+            feature_size=config.num_mel_bins,
+            sampling_rate=config.sampling_rate,
+            subsampling_factor=config.subsampling_factor,
+            subsampling_conv_kernel_size=config.subsampling_conv_kernel_size,
+            subsampling_conv_stride=config.subsampling_conv_stride,
+        )
+
+
 class ParakeetExtractor(ParakeetFeatureExtractor):
     def __init__(self, config: PretrainedConfig) -> None:
         self.config = ExtractorConfig.from_hf_config(config)

From 3e30def7fe46aba97d7552fdae1b5591a7829e2f Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Mon, 23 Feb 2026 07:03:37 -0800
Subject: [PATCH 02/39] move ParakeetConfig and ExtractorConfig to
 vllm/transformers_utils/configs/parakeet.py

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/parakeet.py | 53 ++------------------------
 1 file changed, 3 insertions(+), 50 deletions(-)

diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index c86916b0d5ff..beeba9ddf28c 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -3,42 +3,17 @@
 
 
 from collections.abc import Iterable
-from dataclasses import asdict, dataclass
+from dataclasses import asdict
 
 import numpy as np
 import torch
 import torch.nn as nn
 from transformers import ParakeetEncoder as HFParakeetEncoder
-from transformers import (
-    ParakeetEncoderConfig,
-    ParakeetFeatureExtractor,
-    PretrainedConfig,
-)
+from transformers import ParakeetFeatureExtractor, PretrainedConfig
 
 from vllm.model_executor.layers.activation import ReLUSquaredActivation
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-
-
-class ParakeetConfig(ParakeetEncoderConfig):
-    llm_hidden_size: int
-    projection_hidden_size: int
-    projection_bias: bool
-    projection_eps: float = 1e-5
-    sampling_rate: int
-
-    @staticmethod
-    def from_hf_config(
-        config: PretrainedConfig, *, llm_hidden_size: int, max_model_len: int
-    ) -> "ParakeetConfig":
-        assert isinstance(config, PretrainedConfig)
-        return ParakeetConfig(
-            **config.to_dict(),
-            scale_input=False,
-            attention_bias=False,
-            llm_hidden_size=llm_hidden_size,
-            max_position_embeddings=max_model_len
-            + 1,  # + 1 because it seems like max_model_len+1 can be passed
-        )
+from vllm.transformers_utils.configs.parakeet import ExtractorConfig, ParakeetConfig
 
 
 class ParakeetProjection(nn.Module):
@@ -125,28 +100,6 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         return loaded_params
 
 
-@dataclass(kw_only=True, frozen=True)
-class ExtractorConfig:
-    feature_size: int
-    sampling_rate: int
-    subsampling_factor: int
-    subsampling_conv_kernel_size: int
-    subsampling_conv_stride: int
-    clip_duration_s: int = 30
-    clip_min_duration_s: float = 0.1
-
-    @staticmethod
-    def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
-        assert isinstance(config, PretrainedConfig)
-        return ExtractorConfig(
-            feature_size=config.num_mel_bins,
-            sampling_rate=config.sampling_rate,
-            subsampling_factor=config.subsampling_factor,
-            subsampling_conv_kernel_size=config.subsampling_conv_kernel_size,
-            subsampling_conv_stride=config.subsampling_conv_stride,
-        )
-
-
 class ParakeetExtractor(ParakeetFeatureExtractor):
     def __init__(self, config: PretrainedConfig) -> None:
         self.config = ExtractorConfig.from_hf_config(config)

From b1d4c8d9390cc137ecc0230d88cfcb2d1c622bd8 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Tue, 3 Feb 2026 06:02:03 -0800
Subject: [PATCH 03/39] the model starts at least

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 91 ++++++++-----------
 vllm/model_executor/models/parakeet.py        | 82 ++++++++++-------
 2 files changed, 84 insertions(+), 89 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 66ae76beb8e2..fcef76c39e9d 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -23,7 +23,11 @@
 import torch.nn as nn
 import torchvision.transforms as T
 from PIL import Image
-from transformers import BatchFeature, PretrainedConfig, TensorType
+from transformers import (
+    BatchFeature,
+    PretrainedConfig,
+    TensorType,
+)
 
 from vllm.config import VllmConfig
 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
@@ -44,7 +48,10 @@
 )
 from vllm.model_executor.models.module_mapping import MultiModelKeys
 from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
-from vllm.model_executor.models.parakeet import ParakeetExtractor, ProjectedParakeet
+from vllm.model_executor.models.parakeet import (
+    ParakeetExtractor,
+    ProjectedParakeet,
+)
 from vllm.model_executor.models.radio import RadioModel, calc_seq_lens
 from vllm.model_executor.models.utils import (
     init_vllm_registered_model,
@@ -95,6 +102,9 @@
 # Image.MAX_IMAGE_PIXELS = 300000000  # ~300M pixels
 
 
+AUDIO_CONTEXT = "<so_embedding>"
+
+
 class NanoNemotronVLAudioFeatureInputs(TensorSchema):
     """
     Dimensions:
@@ -1001,8 +1011,10 @@ def _preprocess_audio(
         audio_index = 0
         for idx, part in enumerate(parts):
             if part == AUDIO_CONTEXT:
-                audio_repl = self.get_audio_repl(audios[audio_index])
-                parts[idx] = audio_repl.full
+                audio = audios[audio_index]
+                audio_len = len(audio)
+                num_tokens = extractor.audio_token_count(audio_len)
+                parts[idx] = AUDIO_CONTEXT * num_tokens
                 audio_index += 1
         text = ["".join(parts)]
         audio_inputs = extractor(
@@ -1235,20 +1247,6 @@ def supports_video(self):
     def audio_extractor(self) -> ParakeetExtractor | None:
         return self.get_hf_processor().audio_extractor
 
-    def get_data_parser(self):
-        target_sr = None
-        target_channels = None
-        if extractor := self.audio_extractor:
-            target_sr = extractor.sampling_rate
-            target_channels = 1
-
-        return MultiModalDataParser(
-            video_needs_metadata=True,
-            target_sr=target_sr,
-            target_channels=target_channels,
-            expected_hidden_size=self._get_expected_hidden_size(),
-        )
-
     def get_supported_mm_limits(self):
         video_limit = {"video": None} if self.supports_video else {}
         audio_limit = {"audio": None} if self.audio_extractor is not None else {}
@@ -1383,6 +1381,18 @@ class NanoNemotronVLMultiModalProcessor(
 ):
     """MultiModalProcessor extended for video support"""
 
+    def _get_data_parser(self) -> MultiModalDataParser:
+        target_sr = None
+        target_channels = None
+        if extractor := self.info.audio_extractor:
+            target_sr = extractor.sampling_rate
+            target_channels = 1
+        return MultiModalDataParser(
+            video_needs_metadata=True,
+            target_sr=target_sr,
+            target_channels=target_channels,
+        )
+
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,
@@ -1481,9 +1491,12 @@ def get_video_replacement_internvl(item_idx: int):
                 ),
             ]
 
-        def get_audio_replacement(item_idx: int):
+        def get_audio_replacement(item_idx: int, extractor: ParakeetExtractor):
             audios = mm_items.get_items("audio", AudioProcessorItems)
-            return hf_processor.get_audio_repl(audios.get(item_idx))
+            audio_len = audios.get_audio_length(item_idx)
+            num_tokens = extractor.audio_token_count(audio_len)
+            repl_full = AUDIO_CONTEXT * num_tokens
+            return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
 
         if self.info.audio_extractor is not None:
             prompt_repl = [
@@ -1491,7 +1504,9 @@ def get_audio_replacement(item_idx: int):
                 PromptReplacement(
                     modality="audio",
                     target=AUDIO_CONTEXT,
-                    replacement=get_audio_replacement,
+                    replacement=lambda item_idx: get_audio_replacement(
+                        item_idx, self.info.audio_extractor
+                    ),
                 ),
             ]
 
@@ -1664,7 +1679,6 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         self.ps_version = config.ps_version
         self.image_tag_type = config.image_tag_type
         self.video_pruning_rate = multimodal_config.video_pruning_rate
-
         with self._mark_language_model(vllm_config):
             self.language_model = init_vllm_registered_model(
                 vllm_config=vllm_config,
@@ -1952,43 +1966,12 @@ def _process_audio_input(
         input_audio_features = audio_input.input_audio_features
         feature_attention_mask = audio_input.feature_attention_mask
         target_device = next(self.sound_encoder.parameters()).device
-
-        # When cross-request batching combines audio clips with different
-        # time dimensions, _reduce_data returns a list instead of a stacked
-        # tensor. Pad to the max time dim and stack; the attention mask
-        # already marks valid positions so zero-padding is safe.
-        if isinstance(input_audio_features, list):
-            feature_sizes = [f.shape[-2] for f in input_audio_features]
-            max_t = max(feature_sizes)
-            padded_feats = [
-                torch.nn.functional.pad(feat, (0, 0, 0, max_t - feat_size))
-                for feat, feat_size in zip(
-                    input_audio_features, feature_sizes, strict=True
-                )
-            ]
-            padded_masks = [
-                torch.nn.functional.pad(mask, (0, max_t - mask.shape[-1]))
-                for mask in feature_attention_mask
-            ]
-            input_audio_features = torch.stack(padded_feats)
-            feature_attention_mask = torch.stack(padded_masks)
-
         input_audio_features = input_audio_features.to(
             dtype=self.llm_dtype, device=target_device
         )
         feature_attention_mask = feature_attention_mask.to(device=target_device)
         sound_embeds = self.sound_encoder(input_audio_features, feature_attention_mask)
-
-        valid_input_lens = feature_attention_mask.sum(dim=1)
-        valid_output_lens = self.sound_encoder.encoder._get_subsampling_output_length(
-            valid_input_lens
-        )
-        truncated_embeds = []
-        for i in range(sound_embeds.shape[0]):
-            valid_len = valid_output_lens[i].item()
-            truncated_embeds.append(sound_embeds[i, :valid_len])
-
-        return tuple(truncated_embeds)
+        return tuple(sound_embeds.unbind(dim=0))
 
     def _create_final_video_embeddings(
         self,
diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index beeba9ddf28c..495985d9c8fe 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -2,18 +2,43 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 
+import math
 from collections.abc import Iterable
-from dataclasses import asdict
+from dataclasses import asdict, dataclass
 
-import numpy as np
 import torch
 import torch.nn as nn
 from transformers import ParakeetEncoder as HFParakeetEncoder
-from transformers import ParakeetFeatureExtractor, PretrainedConfig
+from transformers import (
+    ParakeetEncoderConfig,
+    ParakeetFeatureExtractor,
+    PretrainedConfig,
+)
 
 from vllm.model_executor.layers.activation import ReLUSquaredActivation
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.transformers_utils.configs.parakeet import ExtractorConfig, ParakeetConfig
+
+
+class ParakeetConfig(ParakeetEncoderConfig):
+    llm_hidden_size: int
+    projection_hidden_size: int
+    projection_bias: bool
+    projection_eps: float = 1e-5
+    sampling_rate: int
+
+    @staticmethod
+    def from_hf_config(
+        config: PretrainedConfig, *, llm_hidden_size: int, max_model_len: int
+    ) -> "ParakeetConfig":
+        assert isinstance(config, PretrainedConfig)
+        return ParakeetConfig(
+            **config.to_dict(),
+            scale_input=False,
+            attention_bias=False,
+            llm_hidden_size=llm_hidden_size,
+            max_position_embeddings=max_model_len
+            + 1,  # + 1 because it seems like max_model_len+1 can be passed
+        )
 
 
 class ParakeetProjection(nn.Module):
@@ -100,44 +125,31 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         return loaded_params
 
 
+@dataclass(kw_only=True, frozen=True)
+class ExtractorConfig:
+    feature_size: int
+    sampling_rate: int
+    subsampling_factor: int
+
+    @staticmethod
+    def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
+        assert isinstance(config, PretrainedConfig)
+        return ExtractorConfig(
+            feature_size=config.num_mel_bins,
+            sampling_rate=config.sampling_rate,
+            subsampling_factor=config.subsampling_factor,
+        )
+
+
 class ParakeetExtractor(ParakeetFeatureExtractor):
     def __init__(self, config: PretrainedConfig) -> None:
         self.config = ExtractorConfig.from_hf_config(config)
         super().__init__(**asdict(self.config))
-        self._clip_target_samples = int(
-            round(self.config.clip_duration_s * self.sampling_rate)
-        )
-        self._tail_min_samples = int(
-            round(self.config.clip_min_duration_s * self.sampling_rate)
-        )
-
-    def _normalize_audio_length(self, audio_len: int) -> int:
-        # Match mcore's compute_params() logic for clip/minduration handling.
-        target_len = max(audio_len, self._tail_min_samples)
-        tail_remainder = target_len % self._clip_target_samples
-        if 0 < tail_remainder < self._tail_min_samples:
-            padding = self._tail_min_samples - tail_remainder
-            target_len += padding
-        assert isinstance(target_len, int)
-        return target_len
 
     def audio_token_count(self, audio_len: int) -> int:
-        audio_len = self._normalize_audio_length(audio_len)
         num_frames = audio_len // self.hop_length
-        n_tokens = HFParakeetEncoder._get_subsampling_output_length(
-            self, torch.tensor([num_frames], dtype=torch.float)
-        )
-        return max(1, n_tokens.item())
-
-    def __call__(self, raw_speech: list[np.ndarray], *args, **kwargs):
-        padded = []
-        for p in raw_speech:
-            assert p.ndim == 1
-            audio_len = int(p.shape[0])
-            target_len = self._normalize_audio_length(audio_len)
-            p = np.pad(p, (0, target_len - audio_len))
-            padded.append(p)
-        return super().__call__(padded, *args, **kwargs)
+        n_tokens = math.ceil(num_frames / self.config.subsampling_factor)
+        return max(1, n_tokens)
 
     def audio_length(self, audio_tokens: int) -> int:
         return int(audio_tokens * self.config.subsampling_factor * self.hop_length)

From c3443d762bdef060b1a034c3480ed124f435d99d Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Tue, 3 Feb 2026 08:58:46 -0800
Subject: [PATCH 04/39] support bs>1?

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 12 +++++++++++-
 vllm/model_executor/models/parakeet.py         | 11 ++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index fcef76c39e9d..dab0ea3d0033 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1971,7 +1971,17 @@ def _process_audio_input(
         )
         feature_attention_mask = feature_attention_mask.to(device=target_device)
         sound_embeds = self.sound_encoder(input_audio_features, feature_attention_mask)
-        return tuple(sound_embeds.unbind(dim=0))
+
+        valid_input_lens = feature_attention_mask.sum(dim=1)
+        valid_output_lens = self.sound_encoder.encoder._get_subsampling_output_length(
+            valid_input_lens
+        )
+        truncated_embeds = []
+        for i in range(sound_embeds.shape[0]):
+            valid_len = valid_output_lens[i].item()
+            truncated_embeds.append(sound_embeds[i, :valid_len])
+
+        return tuple(truncated_embeds)
 
     def _create_final_video_embeddings(
         self,
diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index 495985d9c8fe..6b15af362792 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 
-import math
 from collections.abc import Iterable
 from dataclasses import asdict, dataclass
 
@@ -130,6 +129,8 @@ class ExtractorConfig:
     feature_size: int
     sampling_rate: int
     subsampling_factor: int
+    subsampling_conv_kernel_size: int
+    subsampling_conv_stride: int
 
     @staticmethod
     def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
@@ -138,6 +139,8 @@ def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
             feature_size=config.num_mel_bins,
             sampling_rate=config.sampling_rate,
             subsampling_factor=config.subsampling_factor,
+            subsampling_conv_kernel_size=config.subsampling_conv_kernel_size,
+            subsampling_conv_stride=config.subsampling_conv_stride,
         )
 
 
@@ -148,8 +151,10 @@ def __init__(self, config: PretrainedConfig) -> None:
 
     def audio_token_count(self, audio_len: int) -> int:
         num_frames = audio_len // self.hop_length
-        n_tokens = math.ceil(num_frames / self.config.subsampling_factor)
-        return max(1, n_tokens)
+        n_tokens = HFParakeetEncoder._get_subsampling_output_length(
+            self, torch.tensor([num_frames], dtype=torch.float)
+        )
+        return max(1, n_tokens.item())
 
     def audio_length(self, audio_tokens: int) -> int:
         return int(audio_tokens * self.config.subsampling_factor * self.hop_length)

From 0672abeb65c2c4a520d2ceee4ee8d2ebf3dd61d1 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Tue, 10 Feb 2026 03:50:22 -0800
Subject: [PATCH 05/39] support bs>1

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index dab0ea3d0033..949b09682e4b 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1966,6 +1966,27 @@ def _process_audio_input(
         input_audio_features = audio_input.input_audio_features
         feature_attention_mask = audio_input.feature_attention_mask
         target_device = next(self.sound_encoder.parameters()).device
+
+        # When cross-request batching combines audio clips with different
+        # time dimensions, _reduce_data returns a list instead of a stacked
+        # tensor. Pad to the max time dim and stack; the attention mask
+        # already marks valid positions so zero-padding is safe.
+        if isinstance(input_audio_features, list):
+            feature_sizes = [f.shape[-2] for f in input_audio_features]
+            max_t = max(feature_sizes)
+            padded_feats = [
+                torch.nn.functional.pad(feat, (0, 0, 0, max_t - feat_size))
+                for feat, feat_size in zip(
+                    input_audio_features, feature_sizes, strict=True
+                )
+            ]
+            padded_masks = [
+                torch.nn.functional.pad(mask, (0, max_t - mask.shape[-1]))
+                for mask in feature_attention_mask
+            ]
+            input_audio_features = torch.stack(padded_feats)
+            feature_attention_mask = torch.stack(padded_masks)
+
         input_audio_features = input_audio_features.to(
             dtype=self.llm_dtype, device=target_device
         )

From fa583b05fb5130eeb1a52ea0a0225b08c739db33 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Wed, 11 Feb 2026 02:30:39 -0800
Subject: [PATCH 06/39] encase with <so_start> and <so_end>

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 20 +++++--------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 949b09682e4b..a7490c6a5c47 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -102,9 +102,6 @@
 # Image.MAX_IMAGE_PIXELS = 300000000  # ~300M pixels
 
 
-AUDIO_CONTEXT = "<so_embedding>"
-
-
 class NanoNemotronVLAudioFeatureInputs(TensorSchema):
     """
     Dimensions:
@@ -1011,10 +1008,8 @@ def _preprocess_audio(
         audio_index = 0
         for idx, part in enumerate(parts):
             if part == AUDIO_CONTEXT:
-                audio = audios[audio_index]
-                audio_len = len(audio)
-                num_tokens = extractor.audio_token_count(audio_len)
-                parts[idx] = AUDIO_CONTEXT * num_tokens
+                audio_repl = self.get_audio_repl(audios[audio_index])
+                parts[idx] = audio_repl.full
                 audio_index += 1
         text = ["".join(parts)]
         audio_inputs = extractor(
@@ -1491,12 +1486,9 @@ def get_video_replacement_internvl(item_idx: int):
                 ),
             ]
 
-        def get_audio_replacement(item_idx: int, extractor: ParakeetExtractor):
+        def get_audio_replacement(item_idx: int):
             audios = mm_items.get_items("audio", AudioProcessorItems)
-            audio_len = audios.get_audio_length(item_idx)
-            num_tokens = extractor.audio_token_count(audio_len)
-            repl_full = AUDIO_CONTEXT * num_tokens
-            return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
+            return hf_processor.get_audio_repl(audios.get(item_idx))
 
         if self.info.audio_extractor is not None:
             prompt_repl = [
@@ -1504,9 +1496,7 @@ def get_audio_replacement(item_idx: int, extractor: ParakeetExtractor):
                 PromptReplacement(
                     modality="audio",
                     target=AUDIO_CONTEXT,
-                    replacement=lambda item_idx: get_audio_replacement(
-                        item_idx, self.info.audio_extractor
-                    ),
+                    replacement=get_audio_replacement,
                 ),
             ]
 

From 47f1772bcd84778a8f6e76220c2856ceb71ff2f1 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Thu, 12 Feb 2026 13:31:22 -0800
Subject: [PATCH 07/39] don't truncate hop_length remainder, pad to 0.1s
 instead

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/parakeet.py | 30 ++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index 6b15af362792..c86916b0d5ff 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -5,6 +5,7 @@
 from collections.abc import Iterable
 from dataclasses import asdict, dataclass
 
+import numpy as np
 import torch
 import torch.nn as nn
 from transformers import ParakeetEncoder as HFParakeetEncoder
@@ -131,6 +132,8 @@ class ExtractorConfig:
     subsampling_factor: int
     subsampling_conv_kernel_size: int
     subsampling_conv_stride: int
+    clip_duration_s: int = 30
+    clip_min_duration_s: float = 0.1
 
     @staticmethod
     def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
@@ -148,13 +151,40 @@ class ParakeetExtractor(ParakeetFeatureExtractor):
     def __init__(self, config: PretrainedConfig) -> None:
         self.config = ExtractorConfig.from_hf_config(config)
         super().__init__(**asdict(self.config))
+        self._clip_target_samples = int(
+            round(self.config.clip_duration_s * self.sampling_rate)
+        )
+        self._tail_min_samples = int(
+            round(self.config.clip_min_duration_s * self.sampling_rate)
+        )
+
+    def _normalize_audio_length(self, audio_len: int) -> int:
+        # Match mcore's compute_params() logic for clip/minduration handling.
+        target_len = max(audio_len, self._tail_min_samples)
+        tail_remainder = target_len % self._clip_target_samples
+        if 0 < tail_remainder < self._tail_min_samples:
+            padding = self._tail_min_samples - tail_remainder
+            target_len += padding
+        assert isinstance(target_len, int)
+        return target_len
 
     def audio_token_count(self, audio_len: int) -> int:
+        audio_len = self._normalize_audio_length(audio_len)
         num_frames = audio_len // self.hop_length
         n_tokens = HFParakeetEncoder._get_subsampling_output_length(
             self, torch.tensor([num_frames], dtype=torch.float)
         )
         return max(1, n_tokens.item())
 
+    def __call__(self, raw_speech: list[np.ndarray], *args, **kwargs):
+        padded = []
+        for p in raw_speech:
+            assert p.ndim == 1
+            audio_len = int(p.shape[0])
+            target_len = self._normalize_audio_length(audio_len)
+            p = np.pad(p, (0, target_len - audio_len))
+            padded.append(p)
+        return super().__call__(padded, *args, **kwargs)
+
     def audio_length(self, audio_tokens: int) -> int:
         return int(audio_tokens * self.config.subsampling_factor * self.hop_length)

From 6d13da59aaf32c6f817479aea6606c75ad9989ad Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Sun, 15 Feb 2026 06:30:43 -0800
Subject: [PATCH 08/39] allow configuring static and dynamic resolution per
 request, via max_num_tiles, for passing images as video frames

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 43 ++++++++++++-------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index a7490c6a5c47..6d0e0c9a5c83 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -13,7 +13,6 @@
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
-from functools import cached_property
 from typing import Annotated, Any, Literal, TypeAlias, TypeVar
 
 import einops
@@ -743,17 +742,26 @@ def _images_to_pixel_values_lst(
             for idx, image in enumerate(images)
         ]
 
+    def use_dynamic_tiler(
+        self, max_num_tiles: int | None = None
+    ) -> DynamicResolutionImageTiler | None:
+        """If `max_num_tiles` is explicitly defined on a request, e.g.:
+        `"mm_processor_kwargs": { "max_num_tiles": 1 }`, force static resolution"""
+        if max_num_tiles is not None:
+            return None
+        return self.dynamic_tiler
+
     def _preprocess_image(
         self,
         text: list[str],
         images: list[Image.Image],
-        max_num_tiles: int,
+        max_num_tiles: int | None,
     ) -> tuple[list[str], dict[str, Any]]:
         if len(images) == 0:
             image_inputs = {}
             return text, image_inputs
 
-        if tiler := self.dynamic_tiler:
+        if tiler := self.use_dynamic_tiler(max_num_tiles):
             sans_images = text[0].replace("<image>", "")
             text_prompt_length = len(
                 self.tokenizer(sans_images, add_special_tokens=False).input_ids
@@ -774,6 +782,7 @@ def _preprocess_image(
                 "num_tokens_per_image": num_tokens_per_image,
             }
         else:
+            max_num_tiles = max_num_tiles or self.max_num_tiles
             pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles)
             image_num_patches = torch.tensor([len(item) for item in pixel_values_lst])
             pixel_values_flat = input_conditioner(
@@ -1037,10 +1046,6 @@ def __call__(
         return_tensors: str | TensorType | None = None,
         max_num_tiles: int | None = None,
     ) -> BatchFeature:
-        # Use default if not provided
-        if max_num_tiles is None:
-            max_num_tiles = self.max_num_tiles
-
         text, images, videos, audios = [
             self._make_batch_input(x) for x in (text, images, videos, audios)
         ]
@@ -1283,16 +1288,17 @@ def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
 class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
     """Basic image-only MultiModalProcessor for InternVL-style models."""
 
-    @cached_property
-    def is_dynamic_tiler(self) -> bool:
-        return self.info.get_hf_processor().dynamic_tiler is not None
+    def use_dynamic_tiler(self, hf_processor_mm_kwargs: dict[str, Any]):
+        return self.info.get_hf_processor().use_dynamic_tiler(
+            hf_processor_mm_kwargs.get("max_num_tiles")
+        )
 
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,
         hf_processor_mm_kwargs: Mapping[str, object],
     ) -> Mapping[str, MultiModalFieldConfig]:
-        if self.is_dynamic_tiler:
+        if self.use_dynamic_tiler(hf_processor_mm_kwargs):
             pixel_values_flat = MultiModalFieldConfig.batched("image")
         else:
             image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -1334,7 +1340,7 @@ def get_replacement_custom(item_idx: int):
 
             if isinstance(images, ImageEmbeddingItems):
                 feature_size = images.get_feature_size(item_idx)
-            elif tiler := hf_processor.dynamic_tiler:
+            elif tiler := self.use_dynamic_tiler(hf_processor_mm_kwargs):
                 image = images.get(item_idx)
                 feature_size = tiler.get_cached_feature_size(image)
             else:
@@ -1844,7 +1850,15 @@ def _parse_and_validate_image_input(
                 data=image_embeds,
             )
 
-        if self.dynamic_resolution:
+        has_static = "image_num_patches" in kwargs
+        has_dynamic = "imgs_sizes" in kwargs and "num_tokens_per_image" in kwargs
+        assert has_static != has_dynamic, (
+            "Expected either static or dynamic resolution in batch, "
+            f"but got keys={list(kwargs.keys())} indicating both"
+        )
+
+        if has_dynamic:
+            assert self.dynamic_resolution
             pixel_values_flat = DynamicResolutionImageTiler.stack(
                 kwargs.pop("pixel_values_flat"), self.patch_size
             )
@@ -2133,8 +2147,7 @@ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
                 image_input = modalities["images"]
                 if image_input["type"] == "image_embeds":
                     image_embeddings = image_input["data"]
-                elif self.dynamic_resolution:
-                    assert image_input["type"] == "pixel_values_dynamic"
+                elif image_input["type"] == "pixel_values_dynamic":
                     image_embeddings = self._process_image_input_dynamic(image_input)
                 else:
                     image_embeddings = self._process_image_input(image_input)

From e527e44b60f0c239ef474ce559d7b470aa5d3f16 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster>
Date: Wed, 18 Feb 2026 06:30:34 -0800
Subject: [PATCH 09/39] initial implementation

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 157 ++++++++++++++++++
 vllm/multimodal/media/audio.py                |  35 ++++
 vllm/multimodal/media/video.py                |  14 +-
 3 files changed, 205 insertions(+), 1 deletion(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 6d0e0c9a5c83..559adafeab15 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -65,9 +65,12 @@
     AudioItem,
     MultiModalDataDict,
     MultiModalFieldConfig,
+    MultiModalInputs,
     MultiModalKwargsItems,
+    MultiModalUUIDDict,
     VideoItem,
 )
+from vllm.multimodal.media.audio import extract_audio_from_video_bytes
 from vllm.multimodal.parse import (
     AudioProcessorItems,
     ImageEmbeddingItems,
@@ -75,6 +78,7 @@
     ImageSize,
     MultiModalDataItems,
     MultiModalDataParser,
+    VideoProcessorItems,
 )
 from vllm.multimodal.processing import BaseDummyInputsBuilder
 from vllm.multimodal.processing.processor import (
@@ -1394,6 +1398,159 @@ def _get_data_parser(self) -> MultiModalDataParser:
             target_channels=target_channels,
         )
 
+    def _extract_audio_from_videos(
+        self,
+        mm_data: MultiModalDataDict,
+        max_audio_duration: float | None = None,
+    ) -> tuple[MultiModalDataDict, list[AudioItem]]:
+        """Extract audio tracks from video bytes in *mm_data*.
+
+        Args:
+            mm_data: The multimodal data dict containing video items.
+            max_audio_duration: If set, only extract the first N seconds
+                of audio from each video.
+
+        Returns:
+            The (possibly augmented) *mm_data* and the list of
+            extracted audio items.
+        """
+        video_items = self._get_data_parser().parse_mm_data(
+            {"video": mm_data["video"]}
+        )
+        videos = video_items.get_items("video", VideoProcessorItems)
+        metadata_list = (
+            videos.metadata if isinstance(videos.metadata, list) else []
+        )
+
+        target_sr = None
+        if extractor := self.info.audio_extractor:
+            target_sr = extractor.sampling_rate
+
+        audio_items: list[AudioItem] = []
+        for idx, metadata in enumerate(metadata_list):
+            video_bytes = (
+                metadata.get("original_video_bytes") if metadata else None
+            )
+            if video_bytes is None:
+                raise ValueError(
+                    "`use_audio_in_video=True` requires the raw video "
+                    "bytes to be available in metadata. Make sure the "
+                    "server was started with "
+                    '--media-io-kwargs \'{"video": '
+                    '{"keep_video_bytes": true}}\' '
+                    f"(missing for video index {idx})."
+                )
+            audio_items.append(
+                extract_audio_from_video_bytes(
+                    video_bytes,
+                    sr=target_sr,
+                    max_duration=max_audio_duration,
+                )
+            )
+            metadata.pop("original_video_bytes", None)
+
+        mm_data = dict(mm_data)
+        mm_data["audio"] = audio_items
+        return mm_data, audio_items
+
+    def apply(
+        self,
+        prompt: str | list[int],
+        mm_data: MultiModalDataDict,
+        hf_processor_mm_kwargs: Mapping[str, object],
+        tokenization_kwargs: Mapping[str, object] | None = None,
+        *,
+        mm_uuids: MultiModalUUIDDict | None = None,
+    ) -> MultiModalInputs:
+        use_audio_in_video = bool(
+            hf_processor_mm_kwargs.get("use_audio_in_video", False)
+        )
+        max_audio_duration = hf_processor_mm_kwargs.get(
+            "max_audio_duration", None
+        )
+        if max_audio_duration is not None:
+            max_audio_duration = float(max_audio_duration)
+
+        # Strip our custom kwargs so they don't reach the HF processor
+        _custom_keys = {"use_audio_in_video", "max_audio_duration"}
+        hf_processor_mm_kwargs = {
+            k: v
+            for k, v in hf_processor_mm_kwargs.items()
+            if k not in _custom_keys
+        }
+
+        if not (
+            use_audio_in_video
+            and "video" in mm_data
+            and "audio" not in mm_data
+        ):
+            return super().apply(
+                prompt,
+                mm_data,
+                hf_processor_mm_kwargs,
+                tokenization_kwargs,
+                mm_uuids=mm_uuids,
+            )
+
+        mm_data, audio_items = self._extract_audio_from_videos(
+            mm_data, max_audio_duration=max_audio_duration
+        )
+
+        if not isinstance(prompt, str):
+            tokenizer = self.info.get_tokenizer()
+            prompt = tokenizer.decode(prompt, skip_special_tokens=False)
+
+        for _ in audio_items:
+            prompt = prompt.replace(
+                "<video>", "<video>" + AUDIO_CONTEXT, 1
+            )
+
+        if tokenization_kwargs is None:
+            tokenization_kwargs = {}
+
+        mm_items = self._to_mm_items(mm_data)
+
+        # Use the non-cached path so that the HF processor receives
+        # both the prompt text (with <so_embedding>) and the audio data
+        # together, letting it handle the audio replacement natively.
+        (
+            prompt_ids,
+            mm_info,
+            is_update_applied,
+        ) = self._apply_hf_processor(
+            prompt,
+            mm_items,
+            hf_processor_mm_kwargs,
+            tokenization_kwargs,
+            mm_uuids=mm_uuids,
+        )
+
+        prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
+            mm_items=mm_items,
+            prompt_ids=prompt_ids,
+            mm_kwargs=mm_info.kwargs,
+            mm_prompt_updates=mm_info.prompt_updates,
+            is_update_applied=is_update_applied,
+        )
+
+        mm_placeholder_ranges = {
+            modality: [item.to_range() for item in placeholders]
+            for modality, placeholders in mm_placeholders.items()
+        }
+
+        t14 = time.time()
+        result = MultiModalInputs(
+            type="multimodal",
+            prompt_token_ids=prompt_ids,
+            mm_kwargs=mm_info.kwargs,
+            mm_hashes=mm_info.hashes,
+            mm_placeholders=mm_placeholder_ranges,
+        )
+        t15 = time.time()
+        print(f"[apply] MultiModalInputs pack took {t15 - t14:.3f} seconds")
+
+        return result
+
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index 3a386c148157..1118e0aa0401 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import base64
+import os
+import tempfile
 from io import BytesIO
 from pathlib import Path
 
@@ -24,6 +26,39 @@
     soundfile = PlaceholderModule("soundfile")  # type: ignore[assignment]
 
 
+def extract_audio_from_video_bytes(
+    data: bytes,
+    sr: float | None = None,
+    max_duration: float | None = None,
+) -> tuple[npt.NDArray, float]:
+    """Extract the audio track from raw video bytes using librosa.
+
+    Args:
+        data: Raw video file bytes (e.g. from an mp4 file).
+        sr: Target sampling rate. If ``None``, the native rate is used.
+        max_duration: If set, only load the first *max_duration* seconds
+            of audio.
+
+    Returns:
+        A tuple of ``(waveform, sample_rate)`` suitable for use as an
+        :class:`AudioItem`.
+    """
+    # MP4 containers require file-path access (soundfile can't read them
+    # from BytesIO).  Write to a temp file and let librosa's audioread
+    # backend handle the container -- no subprocess is spawned, which is
+    # critical to avoid crashing CUDA-active vLLM worker processes.
+    tmp_dir = os.environ.get("TMPDIR", "/tmp")
+    with tempfile.NamedTemporaryFile(
+        suffix=".mp4", delete=False, dir=tmp_dir
+    ) as tmp:
+        tmp.write(data)
+        tmp_path = tmp.name
+    try:
+        return librosa.load(tmp_path, sr=sr, duration=max_duration)
+    finally:
+        os.unlink(tmp_path)
+
+
 class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
     def __init__(self, **kwargs) -> None:
         super().__init__()
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index 00ce9fc30a6c..e24c2154df4d 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -40,13 +40,25 @@ def __init__(
         video_loader_backend = (
             kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
         )
+
+        # When True, the raw video bytes are retained in metadata under
+        # the key ``"original_video_bytes"`` so that downstream processors
+        # can extract additional information (e.g. audio tracks) without
+        # requiring a second read. Opt-in via:
+        #   --media-io-kwargs '{"video": {"keep_video_bytes": true}}'
+        self.keep_video_bytes: bool = bool(kwargs.pop("keep_video_bytes", False))
+
         self.kwargs = kwargs
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
 
     def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
-        return self.video_loader.load_bytes(
+        video, metadata = self.video_loader.load_bytes(
             data, num_frames=self.num_frames, **self.kwargs
         )
+        if self.keep_video_bytes:
+            metadata = dict(metadata)
+            metadata["original_video_bytes"] = data
+        return video, metadata
 
     def load_base64(
         self, media_type: str, data: str

From 72b564681293364032f3bcc7fda3a32b8a7bfc9d Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster>
Date: Wed, 18 Feb 2026 07:04:15 -0800
Subject: [PATCH 10/39] Refactor audio extraction from video bytes to use PyAV
 instead of librosa, improving performance and compatibility with CUDA.
 Cleaned up related code and removed unused parameters. Updated .gitignore to
 include video files for testing.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 33 ++---------
 vllm/multimodal/media/audio.py                | 57 ++++++++++++-------
 2 files changed, 42 insertions(+), 48 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 559adafeab15..eb6b26811438 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1401,15 +1401,9 @@ def _get_data_parser(self) -> MultiModalDataParser:
     def _extract_audio_from_videos(
         self,
         mm_data: MultiModalDataDict,
-        max_audio_duration: float | None = None,
     ) -> tuple[MultiModalDataDict, list[AudioItem]]:
         """Extract audio tracks from video bytes in *mm_data*.
 
-        Args:
-            mm_data: The multimodal data dict containing video items.
-            max_audio_duration: If set, only extract the first N seconds
-                of audio from each video.
-
         Returns:
             The (possibly augmented) *mm_data* and the list of
             extracted audio items.
@@ -1444,7 +1438,6 @@ def _extract_audio_from_videos(
                 extract_audio_from_video_bytes(
                     video_bytes,
                     sr=target_sr,
-                    max_duration=max_audio_duration,
                 )
             )
             metadata.pop("original_video_bytes", None)
@@ -1465,18 +1458,11 @@ def apply(
         use_audio_in_video = bool(
             hf_processor_mm_kwargs.get("use_audio_in_video", False)
         )
-        max_audio_duration = hf_processor_mm_kwargs.get(
-            "max_audio_duration", None
-        )
-        if max_audio_duration is not None:
-            max_audio_duration = float(max_audio_duration)
 
-        # Strip our custom kwargs so they don't reach the HF processor
-        _custom_keys = {"use_audio_in_video", "max_audio_duration"}
         hf_processor_mm_kwargs = {
             k: v
             for k, v in hf_processor_mm_kwargs.items()
-            if k not in _custom_keys
+            if k != "use_audio_in_video"
         }
 
         if not (
@@ -1492,9 +1478,7 @@ def apply(
                 mm_uuids=mm_uuids,
             )
 
-        mm_data, audio_items = self._extract_audio_from_videos(
-            mm_data, max_audio_duration=max_audio_duration
-        )
+        mm_data, audio_items = self._extract_audio_from_videos(mm_data)
 
         if not isinstance(prompt, str):
             tokenizer = self.info.get_tokenizer()
@@ -1510,9 +1494,9 @@ def apply(
 
         mm_items = self._to_mm_items(mm_data)
 
-        # Use the non-cached path so that the HF processor receives
-        # both the prompt text (with <so_embedding>) and the audio data
-        # together, letting it handle the audio replacement natively.
+        # Bypass the cached path: the HF processor must receive the
+        # prompt (with injected <so_embedding>) and the audio data
+        # together so it can perform audio-token replacement natively.
         (
             prompt_ids,
             mm_info,
@@ -1538,18 +1522,13 @@ def apply(
             for modality, placeholders in mm_placeholders.items()
         }
 
-        t14 = time.time()
-        result = MultiModalInputs(
+        return MultiModalInputs(
             type="multimodal",
             prompt_token_ids=prompt_ids,
             mm_kwargs=mm_info.kwargs,
             mm_hashes=mm_info.hashes,
             mm_placeholders=mm_placeholder_ranges,
         )
-        t15 = time.time()
-        print(f"[apply] MultiModalInputs pack took {t15 - t14:.3f} seconds")
-
-        return result
 
     def _get_mm_fields_config(
         self,
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index 1118e0aa0401..d5d0bf3d6495 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -1,11 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import base64
-import os
-import tempfile
 from io import BytesIO
 from pathlib import Path
 
+import numpy as np
 import numpy.typing as npt
 import pybase64
 import torch
@@ -15,6 +14,11 @@
 
 from .base import MediaIO
 
+try:
+    import av
+except ImportError:
+    av = PlaceholderModule("av")  # type: ignore[assignment]
+
 try:
     import librosa
 except ImportError:
@@ -29,34 +33,45 @@
 def extract_audio_from_video_bytes(
     data: bytes,
     sr: float | None = None,
-    max_duration: float | None = None,
 ) -> tuple[npt.NDArray, float]:
-    """Extract the audio track from raw video bytes using librosa.
+    """Extract the audio track from raw video bytes using PyAV.
+
+    PyAV wraps FFmpeg's C libraries in-process — no subprocess is
+    spawned, which is critical to avoid crashing CUDA-active vLLM
+    worker processes.
 
     Args:
         data: Raw video file bytes (e.g. from an mp4 file).
-        sr: Target sampling rate. If ``None``, the native rate is used.
-        max_duration: If set, only load the first *max_duration* seconds
-            of audio.
+        sr: Target sampling rate.  If ``None``, the native rate is used.
 
     Returns:
         A tuple of ``(waveform, sample_rate)`` suitable for use as an
         :class:`AudioItem`.
     """
-    # MP4 containers require file-path access (soundfile can't read them
-    # from BytesIO).  Write to a temp file and let librosa's audioread
-    # backend handle the container -- no subprocess is spawned, which is
-    # critical to avoid crashing CUDA-active vLLM worker processes.
-    tmp_dir = os.environ.get("TMPDIR", "/tmp")
-    with tempfile.NamedTemporaryFile(
-        suffix=".mp4", delete=False, dir=tmp_dir
-    ) as tmp:
-        tmp.write(data)
-        tmp_path = tmp.name
-    try:
-        return librosa.load(tmp_path, sr=sr, duration=max_duration)
-    finally:
-        os.unlink(tmp_path)
+    container = av.open(BytesIO(data))
+    stream = container.streams.audio[0]
+    native_sr = stream.rate
+
+    chunks: list[npt.NDArray] = []
+    for frame in container.decode(audio=0):
+        # to_ndarray() returns shape (channels, samples) for planar
+        # formats and (1, samples) for packed formats.
+        arr = frame.to_ndarray()
+        chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
+    container.close()
+
+    if not chunks:
+        audio = np.array([], dtype=np.float32)
+    else:
+        audio = np.concatenate(chunks).astype(np.float32)
+
+    if sr is not None and sr != native_sr:
+        audio = librosa.resample(
+            audio, orig_sr=float(native_sr), target_sr=float(sr)
+        )
+        return audio, float(sr)
+
+    return audio, float(native_sr)
 
 
 class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):

From a1d506acacdf264c4f6033cc8903449aecac5f45 Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Thu, 19 Feb 2026 15:22:44 +0100
Subject: [PATCH 11/39] Enhance multimodal processing by adding support for
 `mm_processor_kwargs` in various components, allowing for more flexible video
 handling. Updated `.gitignore` to remove exclusions for asset files and added
 error handling for audio extraction from video bytes. Improved video loading
 methods to retain original video bytes when specified.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py                | 41 +++++++++++++++----
 .../model_executor/models/nano_nemotron_vl.py | 23 +++--------
 vllm/multimodal/media/audio.py                |  2 +-
 vllm/multimodal/media/video.py                | 22 +++++-----
 vllm/multimodal/utils.py                      |  3 ++
 5 files changed, 55 insertions(+), 36 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index c48d7bea983c..f39f9d1649bd 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -461,10 +461,15 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
     maximum per prompt.
     """
 
-    def __init__(self, model_config: ModelConfig):
+    def __init__(
+        self,
+        model_config: ModelConfig,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ):
         super().__init__()
 
         self._model_config = model_config
+        self._mm_processor_kwargs = mm_processor_kwargs or {}
 
         self._items_by_modality = defaultdict[str, list[_T]](list)
         # Track original modality for each vision_chunk item (image or video)
@@ -869,7 +874,16 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
-        video = self._connector.fetch_video(video_url=video_url) if video_url else None
+        keep_bytes = bool(
+            self._tracker._mm_processor_kwargs.get("use_audio_in_video")
+        )
+        video = (
+            self._connector.fetch_video(
+                video_url=video_url, keep_video_bytes=keep_bytes
+            )
+            if video_url
+            else None
+        )
 
         placeholder = self._tracker.add("video", (video, uuid))
         self._add_placeholder("video", placeholder)
@@ -1012,8 +1026,15 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
+        keep_bytes = bool(
+            self._tracker._mm_processor_kwargs.get("use_audio_in_video")
+        )
         video = (
-            await self._connector.fetch_video_async(video_url) if video_url else None
+            self._connector.fetch_video_async(
+                video_url=video_url, keep_video_bytes=keep_bytes
+            )
+            if video_url
+            else None
         )
         return video, uuid
 
@@ -1521,14 +1542,17 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:
 def parse_chat_messages(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
-    content_format: ChatTemplateContentFormat,
+    content_format: _ChatTemplateContentFormat,
+    mm_processor_kwargs: dict[str, Any] | None = None,
 ) -> tuple[
     list[ConversationMessage],
     MultiModalDataDict | None,
     MultiModalUUIDDict | None,
 ]:
     conversation: list[ConversationMessage] = []
-    mm_tracker = MultiModalItemTracker(model_config)
+    mm_tracker = MultiModalItemTracker(
+        model_config, mm_processor_kwargs=mm_processor_kwargs
+    )
 
     for msg in messages:
         sub_messages = _parse_chat_message_content(
@@ -1554,14 +1578,17 @@ def parse_chat_messages(
 async def parse_chat_messages_async(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
-    content_format: ChatTemplateContentFormat,
+    content_format: _ChatTemplateContentFormat,
+    mm_processor_kwargs: dict[str, Any] | None = None,
 ) -> tuple[
     list[ConversationMessage],
     MultiModalDataDict | None,
     MultiModalUUIDDict | None,
 ]:
     conversation: list[ConversationMessage] = []
-    mm_tracker = AsyncMultiModalItemTracker(model_config)
+    mm_tracker = AsyncMultiModalItemTracker(
+        model_config, mm_processor_kwargs=mm_processor_kwargs
+    )
 
     for msg in messages:
         sub_messages = _parse_chat_message_content(
diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index eb6b26811438..061437d63705 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1412,35 +1412,23 @@ def _extract_audio_from_videos(
             {"video": mm_data["video"]}
         )
         videos = video_items.get_items("video", VideoProcessorItems)
-        metadata_list = (
-            videos.metadata if isinstance(videos.metadata, list) else []
-        )
+        assert isinstance(videos.metadata, list)
+        metadata_list = videos.metadata
 
         target_sr = None
         if extractor := self.info.audio_extractor:
             target_sr = extractor.sampling_rate
 
         audio_items: list[AudioItem] = []
-        for idx, metadata in enumerate(metadata_list):
-            video_bytes = (
-                metadata.get("original_video_bytes") if metadata else None
-            )
-            if video_bytes is None:
-                raise ValueError(
-                    "`use_audio_in_video=True` requires the raw video "
-                    "bytes to be available in metadata. Make sure the "
-                    "server was started with "
-                    '--media-io-kwargs \'{"video": '
-                    '{"keep_video_bytes": true}}\' '
-                    f"(missing for video index {idx})."
-                )
+        for _, metadata in enumerate(metadata_list):
+            video_bytes = metadata.get("original_video_bytes")
             audio_items.append(
                 extract_audio_from_video_bytes(
                     video_bytes,
                     sr=target_sr,
                 )
             )
-            metadata.pop("original_video_bytes", None)
+            del metadata["original_video_bytes"]
 
         mm_data = dict(mm_data)
         mm_data["audio"] = audio_items
@@ -1758,6 +1746,7 @@ def get_dummy_mm_data(
             dummy_video = {}
 
         if extractor := self.info.audio_extractor:
+            assert extractor is not None
             num_audios = mm_counts.get("audio", 0)
             audio_overrides = mm_options.get("audio") if mm_options else None
             tokens_per_audio = max(1, seq_len // max(num_audios, 1))
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index d5d0bf3d6495..c758e95b1128 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -61,7 +61,7 @@ def extract_audio_from_video_bytes(
     container.close()
 
     if not chunks:
-        audio = np.array([], dtype=np.float32)
+        raise ValueError("No audio found in the video.")
     else:
         audio = np.concatenate(chunks).astype(np.float32)
 
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index e24c2154df4d..cac107d8228d 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -41,21 +41,16 @@ def __init__(
             kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
         )
 
-        # When True, the raw video bytes are retained in metadata under
-        # the key ``"original_video_bytes"`` so that downstream processors
-        # can extract additional information (e.g. audio tracks) without
-        # requiring a second read. Opt-in via:
-        #   --media-io-kwargs '{"video": {"keep_video_bytes": true}}'
-        self.keep_video_bytes: bool = bool(kwargs.pop("keep_video_bytes", False))
-
         self.kwargs = kwargs
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
 
-    def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
+    def load_bytes(
+        self, data: bytes, *, keep_video_bytes: bool = False
+    ) -> tuple[npt.NDArray, dict[str, Any]]:
         video, metadata = self.video_loader.load_bytes(
             data, num_frames=self.num_frames, **self.kwargs
         )
-        if self.keep_video_bytes:
+        if keep_video_bytes:
             metadata = dict(metadata)
             metadata["original_video_bytes"] = data
         return video, metadata
@@ -70,7 +65,10 @@ def load_base64(
             )
 
             return np.stack(
-                [np.asarray(load_frame(frame_data)) for frame_data in data.split(",")]
+                [
+                    np.asarray(load_frame(frame_data))
+                    for frame_data in data.split(",")
+                ]
             ), {}
 
         return self.load_bytes(base64.b64decode(data))
@@ -95,7 +93,9 @@ def encode_base64(
                 image_format=video_format,
             )
 
-            return ",".join(encode_frame(Image.fromarray(frame)) for frame in video)
+            return ",".join(
+                encode_frame(Image.fromarray(frame)) for frame in video
+            )
 
         msg = "Only JPEG format is supported for now."
         raise NotImplementedError(msg)
diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index 886756c99da9..8b5c853f9b61 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -2,8 +2,11 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import mimetypes
+import warnings
+from functools import partial
 from collections import defaultdict
 from collections.abc import Generator, Sequence
+from concurrent.futures import ThreadPoolExecutor
 from itertools import groupby
 from typing import TYPE_CHECKING, Any
 

From 57a5b2069c47563ab40a7f7284d7ddf1989e6001 Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Thu, 19 Feb 2026 15:24:51 +0100
Subject: [PATCH 12/39] Ensure audio extractor is not None in
 NanoNemotronVLMultiModalProcessor and remove redundant assertion in
 NanoNemotronVLDummyInputsBuilder for improved clarity and consistency.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 061437d63705..31163cad1d73 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1417,6 +1417,7 @@ def _extract_audio_from_videos(
 
         target_sr = None
         if extractor := self.info.audio_extractor:
+            assert extractor is not None
             target_sr = extractor.sampling_rate
 
         audio_items: list[AudioItem] = []
@@ -1746,7 +1747,6 @@ def get_dummy_mm_data(
             dummy_video = {}
 
         if extractor := self.info.audio_extractor:
-            assert extractor is not None
             num_audios = mm_counts.get("audio", 0)
             audio_overrides = mm_options.get("audio") if mm_options else None
             tokens_per_audio = max(1, seq_len // max(num_audios, 1))

From 24f739351227e1543ee92bcca4377896c859e608 Mon Sep 17 00:00:00 2001
From: root <root@pool0-03748.cm.cluster>
Date: Thu, 19 Feb 2026 07:41:10 -0800
Subject: [PATCH 13/39] Enhance error handling for audio extraction and video
 loading in multimodal processing. Added checks for empty video bytes in
 `extract_audio_from_video_bytes` and `NanoNemotronVLMultiModalProcessor`,
 ensuring proper configuration with `keep_video_bytes`. Updated `VideoMediaIO`
 to retain original video bytes based on the new parameter.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py |  7 +++++++
 vllm/multimodal/media/audio.py                 | 14 +++++++++++++-
 vllm/multimodal/media/video.py                 |  3 ++-
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 31163cad1d73..130ef153519c 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1423,6 +1423,13 @@ def _extract_audio_from_videos(
         audio_items: list[AudioItem] = []
         for _, metadata in enumerate(metadata_list):
             video_bytes = metadata.get("original_video_bytes")
+            if video_bytes is None or len(video_bytes) == 0:
+                raise ValueError(
+                    "Cannot extract audio from video: original_video_bytes is "
+                    "missing or empty. When using use_audio_in_video=True, "
+                    "video must be loaded with keep_video_bytes=True (e.g. via "
+                    "the chat API with a model that sets use_audio_in_video)."
+                )
             audio_items.append(
                 extract_audio_from_video_bytes(
                     video_bytes,
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index c758e95b1128..521d10414766 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -48,7 +48,19 @@ def extract_audio_from_video_bytes(
         A tuple of ``(waveform, sample_rate)`` suitable for use as an
         :class:`AudioItem`.
     """
-    container = av.open(BytesIO(data))
+    if data is None or len(data) == 0:
+        raise ValueError(
+            "Cannot extract audio: video bytes are missing or empty. "
+            "Ensure video was loaded with keep_video_bytes=True for "
+            "audio-in-video extraction."
+        )
+    try:
+        container = av.open(BytesIO(data))
+    except Exception as e:
+        raise ValueError(
+            "Invalid or corrupted video data when extracting audio. "
+            "Ensure the input is valid video bytes (e.g. a complete MP4)."
+        ) from e
     stream = container.streams.audio[0]
     native_sr = stream.rate
 
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index cac107d8228d..273398bd8e60 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -40,6 +40,7 @@ def __init__(
         video_loader_backend = (
             kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
         )
+        self.keep_video_bytes = kwargs.pop("keep_video_bytes", False)
 
         self.kwargs = kwargs
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
@@ -50,7 +51,7 @@ def load_bytes(
         video, metadata = self.video_loader.load_bytes(
             data, num_frames=self.num_frames, **self.kwargs
         )
-        if keep_video_bytes:
+        if self.keep_video_bytes:
             metadata = dict(metadata)
             metadata["original_video_bytes"] = data
         return video, metadata

From 409a0e4fb4f834975808cb5d4eed2f49539b167e Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Thu, 19 Feb 2026 17:09:49 +0100
Subject: [PATCH 14/39] Refactor video loading methods in `MediaConnector` and
 `VideoMediaIO` for improved readability. Removed redundant assertions and
 streamlined parameter handling for video bytes. Updated `load_bytes` method
 to simplify its signature.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 3 +--
 vllm/multimodal/media/video.py                 | 4 +---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 130ef153519c..17b5c8157eaf 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1417,11 +1417,10 @@ def _extract_audio_from_videos(
 
         target_sr = None
         if extractor := self.info.audio_extractor:
-            assert extractor is not None
             target_sr = extractor.sampling_rate
 
         audio_items: list[AudioItem] = []
-        for _, metadata in enumerate(metadata_list):
+        for metadata in metadata_list:
             video_bytes = metadata.get("original_video_bytes")
             if video_bytes is None or len(video_bytes) == 0:
                 raise ValueError(
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index 273398bd8e60..142a57b5bb2d 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -45,9 +45,7 @@ def __init__(
         self.kwargs = kwargs
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
 
-    def load_bytes(
-        self, data: bytes, *, keep_video_bytes: bool = False
-    ) -> tuple[npt.NDArray, dict[str, Any]]:
+    def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
         video, metadata = self.video_loader.load_bytes(
             data, num_frames=self.num_frames, **self.kwargs
         )

From 586615b60432651df044245798b766ae8f0be947 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster>
Date: Thu, 19 Feb 2026 08:49:16 -0800
Subject: [PATCH 15/39] Refactor audio handling in  by removing placeholder
 imports and directly importing necessary libraries. Simplify  method calls by
 eliminating redundant parameters. Clean up  by removing unnecessary prompt
 type checks for improved clarity.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py |  4 ----
 vllm/multimodal/media/audio.py                 | 18 +++---------------
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 17b5c8157eaf..62da060740ec 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1475,10 +1475,6 @@ def apply(
 
         mm_data, audio_items = self._extract_audio_from_videos(mm_data)
 
-        if not isinstance(prompt, str):
-            tokenizer = self.info.get_tokenizer()
-            prompt = tokenizer.decode(prompt, skip_special_tokens=False)
-
         for _ in audio_items:
             prompt = prompt.replace(
                 "<video>", "<video>" + AUDIO_CONTEXT, 1
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index 521d10414766..28f6b69103ec 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -9,25 +9,13 @@
 import pybase64
 import torch
 
-from vllm.utils.import_utils import PlaceholderModule
 from vllm.utils.serial_utils import tensor2base64
 
 from .base import MediaIO
 
-try:
-    import av
-except ImportError:
-    av = PlaceholderModule("av")  # type: ignore[assignment]
-
-try:
-    import librosa
-except ImportError:
-    librosa = PlaceholderModule("librosa")  # type: ignore[assignment]
-
-try:
-    import soundfile
-except ImportError:
-    soundfile = PlaceholderModule("soundfile")  # type: ignore[assignment]
+import av
+import librosa
+import soundfile
 
 
 def extract_audio_from_video_bytes(

From f06c9b785be45dd25f00733129e54631dff8baf6 Mon Sep 17 00:00:00 2001
From: root <root@pool0-03748.cm.cluster>
Date: Thu, 19 Feb 2026 09:05:22 -0800
Subject: [PATCH 16/39] Enhance prompt handling in  by adding a check for
 prompt type. If the prompt is not a string, it is decoded using the tokenizer
 to ensure compatibility with audio processing. This improves flexibility in
 handling different prompt formats.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 62da060740ec..17b5c8157eaf 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1475,6 +1475,10 @@ def apply(
 
         mm_data, audio_items = self._extract_audio_from_videos(mm_data)
 
+        if not isinstance(prompt, str):
+            tokenizer = self.info.get_tokenizer()
+            prompt = tokenizer.decode(prompt, skip_special_tokens=False)
+
         for _ in audio_items:
             prompt = prompt.replace(
                 "<video>", "<video>" + AUDIO_CONTEXT, 1

From 21928cee0ac188a4a674353c91aba32d02dfb763 Mon Sep 17 00:00:00 2001
From: root <root@pool0-03748.cm.cluster>
Date: Thu, 19 Feb 2026 09:09:48 -0800
Subject: [PATCH 17/39] Refactor code for improved readability by consolidating
 multi-line statements into single lines across various modules. This includes
 adjustments in , , , , , and  to enhance clarity and maintainability.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py                 |  4 +---
 vllm/model_executor/models/nano_nemotron_vl.py | 18 ++++--------------
 vllm/multimodal/media/audio.py                 | 11 ++++-------
 vllm/multimodal/media/video.py                 |  9 ++-------
 4 files changed, 11 insertions(+), 31 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index f39f9d1649bd..7cf5992e182f 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -874,9 +874,7 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
-        keep_bytes = bool(
-            self._tracker._mm_processor_kwargs.get("use_audio_in_video")
-        )
+        keep_bytes = bool(self._tracker._mm_processor_kwargs.get("use_audio_in_video"))
         video = (
             self._connector.fetch_video(
                 video_url=video_url, keep_video_bytes=keep_bytes
diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 17b5c8157eaf..db6a914accbd 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1408,9 +1408,7 @@ def _extract_audio_from_videos(
             The (possibly augmented) *mm_data* and the list of
             extracted audio items.
         """
-        video_items = self._get_data_parser().parse_mm_data(
-            {"video": mm_data["video"]}
-        )
+        video_items = self._get_data_parser().parse_mm_data({"video": mm_data["video"]})
         videos = video_items.get_items("video", VideoProcessorItems)
         assert isinstance(videos.metadata, list)
         metadata_list = videos.metadata
@@ -1455,16 +1453,10 @@ def apply(
         )
 
         hf_processor_mm_kwargs = {
-            k: v
-            for k, v in hf_processor_mm_kwargs.items()
-            if k != "use_audio_in_video"
+            k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
         }
 
-        if not (
-            use_audio_in_video
-            and "video" in mm_data
-            and "audio" not in mm_data
-        ):
+        if not (use_audio_in_video and "video" in mm_data and "audio" not in mm_data):
             return super().apply(
                 prompt,
                 mm_data,
@@ -1480,9 +1472,7 @@ def apply(
             prompt = tokenizer.decode(prompt, skip_special_tokens=False)
 
         for _ in audio_items:
-            prompt = prompt.replace(
-                "<video>", "<video>" + AUDIO_CONTEXT, 1
-            )
+            prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
 
         if tokenization_kwargs is None:
             tokenization_kwargs = {}
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index 28f6b69103ec..e6dcbb7ac30c 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -4,19 +4,18 @@
 from io import BytesIO
 from pathlib import Path
 
+import av
+import librosa
 import numpy as np
 import numpy.typing as npt
 import pybase64
+import soundfile
 import torch
 
 from vllm.utils.serial_utils import tensor2base64
 
 from .base import MediaIO
 
-import av
-import librosa
-import soundfile
-
 
 def extract_audio_from_video_bytes(
     data: bytes,
@@ -66,9 +65,7 @@ def extract_audio_from_video_bytes(
         audio = np.concatenate(chunks).astype(np.float32)
 
     if sr is not None and sr != native_sr:
-        audio = librosa.resample(
-            audio, orig_sr=float(native_sr), target_sr=float(sr)
-        )
+        audio = librosa.resample(audio, orig_sr=float(native_sr), target_sr=float(sr))
         return audio, float(sr)
 
     return audio, float(native_sr)
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index 142a57b5bb2d..7a2b8893db74 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -64,10 +64,7 @@ def load_base64(
             )
 
             return np.stack(
-                [
-                    np.asarray(load_frame(frame_data))
-                    for frame_data in data.split(",")
-                ]
+                [np.asarray(load_frame(frame_data)) for frame_data in data.split(",")]
             ), {}
 
         return self.load_bytes(base64.b64decode(data))
@@ -92,9 +89,7 @@ def encode_base64(
                 image_format=video_format,
             )
 
-            return ",".join(
-                encode_frame(Image.fromarray(frame)) for frame in video
-            )
+            return ",".join(encode_frame(Image.fromarray(frame)) for frame in video)
 
         msg = "Only JPEG format is supported for now."
         raise NotImplementedError(msg)

From 93359b6f42e27e2295c6f645cd66e87073682362 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster>
Date: Wed, 25 Feb 2026 01:31:31 -0800
Subject: [PATCH 18/39] add av to audio setup

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 556a511a3429..f31b4cf24f7e 100644
--- a/setup.py
+++ b/setup.py
@@ -1056,6 +1056,7 @@ def _read_requirements(filename: str) -> list[str]:
             "scipy",
             "soundfile",
             "mistral_common[audio]",
+            "av",
         ],  # Required for audio processing
         "video": [],  # Kept for backwards compatibility
         "flashinfer": [],  # Kept for backwards compatibility

From 854b6c3f178d11457fe276a53ac8340892e3e03c Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Wed, 25 Feb 2026 14:54:31 +0100
Subject: [PATCH 19/39] Reverted unnecessary changes; Fixed video loading

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 55 ++++++-------------
 vllm/model_executor/models/parakeet.py        | 53 +-----------------
 vllm/multimodal/media/connector.py            | 14 ++++-
 vllm/multimodal/utils.py                      |  2 -
 4 files changed, 33 insertions(+), 91 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index db6a914accbd..400862290b1d 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -13,6 +13,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
+from functools import cached_property
 from typing import Annotated, Any, Literal, TypeAlias, TypeVar
 
 import einops
@@ -22,11 +23,7 @@
 import torch.nn as nn
 import torchvision.transforms as T
 from PIL import Image
-from transformers import (
-    BatchFeature,
-    PretrainedConfig,
-    TensorType,
-)
+from transformers import BatchFeature, PretrainedConfig, TensorType
 
 from vllm.config import VllmConfig
 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
@@ -47,10 +44,7 @@
 )
 from vllm.model_executor.models.module_mapping import MultiModelKeys
 from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
-from vllm.model_executor.models.parakeet import (
-    ParakeetExtractor,
-    ProjectedParakeet,
-)
+from vllm.model_executor.models.parakeet import ParakeetExtractor, ProjectedParakeet
 from vllm.model_executor.models.radio import RadioModel, calc_seq_lens
 from vllm.model_executor.models.utils import (
     init_vllm_registered_model,
@@ -746,26 +740,17 @@ def _images_to_pixel_values_lst(
             for idx, image in enumerate(images)
         ]
 
-    def use_dynamic_tiler(
-        self, max_num_tiles: int | None = None
-    ) -> DynamicResolutionImageTiler | None:
-        """If `max_num_tiles` is explicitly defined on a request, e.g.:
-        `"mm_processor_kwargs": { "max_num_tiles": 1 }`, force static resolution"""
-        if max_num_tiles is not None:
-            return None
-        return self.dynamic_tiler
-
     def _preprocess_image(
         self,
         text: list[str],
         images: list[Image.Image],
-        max_num_tiles: int | None,
+        max_num_tiles: int,
     ) -> tuple[list[str], dict[str, Any]]:
         if len(images) == 0:
             image_inputs = {}
             return text, image_inputs
 
-        if tiler := self.use_dynamic_tiler(max_num_tiles):
+        if tiler := self.dynamic_tiler:
             sans_images = text[0].replace("<image>", "")
             text_prompt_length = len(
                 self.tokenizer(sans_images, add_special_tokens=False).input_ids
@@ -786,7 +771,6 @@ def _preprocess_image(
                 "num_tokens_per_image": num_tokens_per_image,
             }
         else:
-            max_num_tiles = max_num_tiles or self.max_num_tiles
             pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles)
             image_num_patches = torch.tensor([len(item) for item in pixel_values_lst])
             pixel_values_flat = input_conditioner(
@@ -1050,6 +1034,10 @@ def __call__(
         return_tensors: str | TensorType | None = None,
         max_num_tiles: int | None = None,
     ) -> BatchFeature:
+        # Use default if not provided
+        if max_num_tiles is None:
+            max_num_tiles = self.max_num_tiles
+
         text, images, videos, audios = [
             self._make_batch_input(x) for x in (text, images, videos, audios)
         ]
@@ -1292,17 +1280,16 @@ def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
 class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
     """Basic image-only MultiModalProcessor for InternVL-style models."""
 
-    def use_dynamic_tiler(self, hf_processor_mm_kwargs: dict[str, Any]):
-        return self.info.get_hf_processor().use_dynamic_tiler(
-            hf_processor_mm_kwargs.get("max_num_tiles")
-        )
+    @cached_property
+    def is_dynamic_tiler(self) -> bool:
+        return self.info.get_hf_processor().dynamic_tiler is not None
 
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,
         hf_processor_mm_kwargs: Mapping[str, object],
     ) -> Mapping[str, MultiModalFieldConfig]:
-        if self.use_dynamic_tiler(hf_processor_mm_kwargs):
+        if self.is_dynamic_tiler:
             pixel_values_flat = MultiModalFieldConfig.batched("image")
         else:
             image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -1344,7 +1331,7 @@ def get_replacement_custom(item_idx: int):
 
             if isinstance(images, ImageEmbeddingItems):
                 feature_size = images.get_feature_size(item_idx)
-            elif tiler := self.use_dynamic_tiler(hf_processor_mm_kwargs):
+            elif tiler := hf_processor.dynamic_tiler:
                 image = images.get(item_idx)
                 feature_size = tiler.get_cached_feature_size(image)
             else:
@@ -1796,6 +1783,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         self.ps_version = config.ps_version
         self.image_tag_type = config.image_tag_type
         self.video_pruning_rate = multimodal_config.video_pruning_rate
+
         with self._mark_language_model(vllm_config):
             self.language_model = init_vllm_registered_model(
                 vllm_config=vllm_config,
@@ -1971,15 +1959,7 @@ def _parse_and_validate_image_input(
                 data=image_embeds,
             )
 
-        has_static = "image_num_patches" in kwargs
-        has_dynamic = "imgs_sizes" in kwargs and "num_tokens_per_image" in kwargs
-        assert has_static != has_dynamic, (
-            "Expected either static or dynamic resolution in batch, "
-            f"but got keys={list(kwargs.keys())} indicating both"
-        )
-
-        if has_dynamic:
-            assert self.dynamic_resolution
+        if self.dynamic_resolution:
             pixel_values_flat = DynamicResolutionImageTiler.stack(
                 kwargs.pop("pixel_values_flat"), self.patch_size
             )
@@ -2268,7 +2248,8 @@ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
                 image_input = modalities["images"]
                 if image_input["type"] == "image_embeds":
                     image_embeddings = image_input["data"]
-                elif image_input["type"] == "pixel_values_dynamic":
+                elif self.dynamic_resolution:
+                    assert image_input["type"] == "pixel_values_dynamic"
                     image_embeddings = self._process_image_input_dynamic(image_input)
                 else:
                     image_embeddings = self._process_image_input(image_input)
diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index c86916b0d5ff..beeba9ddf28c 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -3,42 +3,17 @@
 
 
 from collections.abc import Iterable
-from dataclasses import asdict, dataclass
+from dataclasses import asdict
 
 import numpy as np
 import torch
 import torch.nn as nn
 from transformers import ParakeetEncoder as HFParakeetEncoder
-from transformers import (
-    ParakeetEncoderConfig,
-    ParakeetFeatureExtractor,
-    PretrainedConfig,
-)
+from transformers import ParakeetFeatureExtractor, PretrainedConfig
 
 from vllm.model_executor.layers.activation import ReLUSquaredActivation
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-
-
-class ParakeetConfig(ParakeetEncoderConfig):
-    llm_hidden_size: int
-    projection_hidden_size: int
-    projection_bias: bool
-    projection_eps: float = 1e-5
-    sampling_rate: int
-
-    @staticmethod
-    def from_hf_config(
-        config: PretrainedConfig, *, llm_hidden_size: int, max_model_len: int
-    ) -> "ParakeetConfig":
-        assert isinstance(config, PretrainedConfig)
-        return ParakeetConfig(
-            **config.to_dict(),
-            scale_input=False,
-            attention_bias=False,
-            llm_hidden_size=llm_hidden_size,
-            max_position_embeddings=max_model_len
-            + 1,  # + 1 because it seems like max_model_len+1 can be passed
-        )
+from vllm.transformers_utils.configs.parakeet import ExtractorConfig, ParakeetConfig
 
 
 class ParakeetProjection(nn.Module):
@@ -125,28 +100,6 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         return loaded_params
 
 
-@dataclass(kw_only=True, frozen=True)
-class ExtractorConfig:
-    feature_size: int
-    sampling_rate: int
-    subsampling_factor: int
-    subsampling_conv_kernel_size: int
-    subsampling_conv_stride: int
-    clip_duration_s: int = 30
-    clip_min_duration_s: float = 0.1
-
-    @staticmethod
-    def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":
-        assert isinstance(config, PretrainedConfig)
-        return ExtractorConfig(
-            feature_size=config.num_mel_bins,
-            sampling_rate=config.sampling_rate,
-            subsampling_factor=config.subsampling_factor,
-            subsampling_conv_kernel_size=config.subsampling_conv_kernel_size,
-            subsampling_conv_stride=config.subsampling_conv_stride,
-        )
-
-
 class ParakeetExtractor(ParakeetFeatureExtractor):
     def __init__(self, config: PretrainedConfig) -> None:
         self.config = ExtractorConfig.from_hf_config(config)
diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py
index 784a4ca35abf..df1b2b1c1da4 100644
--- a/vllm/multimodal/media/connector.py
+++ b/vllm/multimodal/media/connector.py
@@ -283,6 +283,7 @@ def fetch_video(
         video_url: str,
         *,
         image_mode: str = "RGB",
+        keep_video_bytes: bool = False,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         """
         Load video from an HTTP or base64 data URL.
@@ -290,7 +291,11 @@ def fetch_video(
         image_io = ImageMediaIO(
             image_mode=image_mode, **self.media_io_kwargs.get("image", {})
         )
-        video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
+
+        video_kwargs = dict(self.media_io_kwargs.get("video", {}))
+        video_kwargs["keep_video_bytes"] = keep_video_bytes
+
+        video_io = VideoMediaIO(image_io, **video_kwargs)
 
         return self.load_from_url(
             video_url,
@@ -303,6 +308,7 @@ async def fetch_video_async(
         video_url: str,
         *,
         image_mode: str = "RGB",
+        keep_video_bytes: bool = False,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         """
         Asynchronously load video from an HTTP or base64 data URL.
@@ -312,7 +318,11 @@ async def fetch_video_async(
         image_io = ImageMediaIO(
             image_mode=image_mode, **self.media_io_kwargs.get("image", {})
         )
-        video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
+
+        video_kwargs = dict(self.media_io_kwargs.get("video", {}))
+        video_kwargs["keep_video_bytes"] = keep_video_bytes
+
+        video_io = VideoMediaIO(image_io, **video_kwargs)
 
         return await self.load_from_url_async(
             video_url,
diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index 8b5c853f9b61..fb0cedadb814 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -3,10 +3,8 @@
 
 import mimetypes
 import warnings
-from functools import partial
 from collections import defaultdict
 from collections.abc import Generator, Sequence
-from concurrent.futures import ThreadPoolExecutor
 from itertools import groupby
 from typing import TYPE_CHECKING, Any
 

From 94f23b028cf63bbf4a5ee718d2e05c6b38b07e9c Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Wed, 25 Feb 2026 14:58:57 +0100
Subject: [PATCH 20/39] Add get_data_parser method to
 NanoNemotronVLProcessingInfo and update data parser usage in
 NanoNemotronVLMultiModalProcessor

This commit introduces a new method, get_data_parser, in the NanoNemotronVLProcessingInfo class to streamline the creation of MultiModalDataParser instances. Additionally, it updates the NanoNemotronVLMultiModalProcessor to utilize the new data parser method for improved clarity and maintainability in audio extraction from video data.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 400862290b1d..29d3de0a169d 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1239,6 +1239,20 @@ def supports_video(self):
     def audio_extractor(self) -> ParakeetExtractor | None:
         return self.get_hf_processor().audio_extractor
 
+    def get_data_parser(self):
+        target_sr = None
+        target_channels = None
+        if extractor := self.audio_extractor:
+            target_sr = extractor.sampling_rate
+            target_channels = 1
+
+        return MultiModalDataParser(
+            video_needs_metadata=True,
+            target_sr=target_sr,
+            target_channels=target_channels,
+            expected_hidden_size=self._get_expected_hidden_size(),
+        )
+
     def get_supported_mm_limits(self):
         video_limit = {"video": None} if self.supports_video else {}
         audio_limit = {"audio": None} if self.audio_extractor is not None else {}
@@ -1373,18 +1387,6 @@ class NanoNemotronVLMultiModalProcessor(
 ):
     """MultiModalProcessor extended for video support"""
 
-    def _get_data_parser(self) -> MultiModalDataParser:
-        target_sr = None
-        target_channels = None
-        if extractor := self.info.audio_extractor:
-            target_sr = extractor.sampling_rate
-            target_channels = 1
-        return MultiModalDataParser(
-            video_needs_metadata=True,
-            target_sr=target_sr,
-            target_channels=target_channels,
-        )
-
     def _extract_audio_from_videos(
         self,
         mm_data: MultiModalDataDict,
@@ -1395,7 +1397,7 @@ def _extract_audio_from_videos(
             The (possibly augmented) *mm_data* and the list of
             extracted audio items.
         """
-        video_items = self._get_data_parser().parse_mm_data({"video": mm_data["video"]})
+        video_items = self.data_parser.parse_mm_data({"video": mm_data["video"]})
         videos = video_items.get_items("video", VideoProcessorItems)
         assert isinstance(videos.metadata, list)
         metadata_list = videos.metadata

From 2f476ef71c32e0a9a2fdaa5786ceb6c6b3e5a17f Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Wed, 25 Feb 2026 15:08:36 +0100
Subject: [PATCH 21/39] Refactor chat message parsing by updating content
 format type annotations for consistency and improving readability in
 AsyncMultiModalContentParser. Consolidated multi-line statements into single
 lines where applicable.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index 7cf5992e182f..589ec92818fd 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -1024,9 +1024,7 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
-        keep_bytes = bool(
-            self._tracker._mm_processor_kwargs.get("use_audio_in_video")
-        )
+        keep_bytes = bool(self._tracker._mm_processor_kwargs.get("use_audio_in_video"))
         video = (
             self._connector.fetch_video_async(
                 video_url=video_url, keep_video_bytes=keep_bytes
@@ -1540,7 +1538,7 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:
 def parse_chat_messages(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
-    content_format: _ChatTemplateContentFormat,
+    content_format: ChatTemplateContentFormat,
     mm_processor_kwargs: dict[str, Any] | None = None,
 ) -> tuple[
     list[ConversationMessage],
@@ -1576,7 +1574,7 @@ def parse_chat_messages(
 async def parse_chat_messages_async(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
-    content_format: _ChatTemplateContentFormat,
+    content_format: ChatTemplateContentFormat,
     mm_processor_kwargs: dict[str, Any] | None = None,
 ) -> tuple[
     list[ConversationMessage],

From 4eae1579a5991a72dbaca87f157324d686f52905 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster>
Date: Wed, 25 Feb 2026 07:34:22 -0800
Subject: [PATCH 22/39] Refactor video handling in AsyncMultiModalContentParser
 and NanoNemotronVLMultiModalProcessor. Updated method signatures to improve
 clarity and consistency in parameter usage. Enhanced MediaConnector to handle
 video byte retention more flexibly by checking existing parameters. This
 improves maintainability and readability across the modules.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py                |  2 +-
 .../model_executor/models/nano_nemotron_vl.py | 46 ++++++++++---------
 vllm/multimodal/media/connector.py            |  8 +++-
 3 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index 589ec92818fd..b65f859a3659 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -1026,7 +1026,7 @@ def parse_input_audio(
     async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
         keep_bytes = bool(self._tracker._mm_processor_kwargs.get("use_audio_in_video"))
         video = (
-            self._connector.fetch_video_async(
+            await self._connector.fetch_video_async(
                 video_url=video_url, keep_video_bytes=keep_bytes
             )
             if video_url
diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 29d3de0a169d..464b3a270e22 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -61,7 +61,6 @@
     MultiModalFieldConfig,
     MultiModalInputs,
     MultiModalKwargsItems,
-    MultiModalUUIDDict,
     VideoItem,
 )
 from vllm.multimodal.media.audio import extract_audio_from_video_bytes
@@ -72,6 +71,7 @@
     ImageSize,
     MultiModalDataItems,
     MultiModalDataParser,
+    MultiModalUUIDItems,
     VideoProcessorItems,
 )
 from vllm.multimodal.processing import BaseDummyInputsBuilder
@@ -1389,16 +1389,15 @@ class NanoNemotronVLMultiModalProcessor(
 
     def _extract_audio_from_videos(
         self,
-        mm_data: MultiModalDataDict,
-    ) -> tuple[MultiModalDataDict, list[AudioItem]]:
-        """Extract audio tracks from video bytes in *mm_data*.
+        mm_items: MultiModalDataItems,
+    ) -> tuple[MultiModalDataItems, list[AudioItem]]:
+        """Extract audio tracks from video bytes in *mm_items*.
 
         Returns:
-            The (possibly augmented) *mm_data* and the list of
+            The augmented *mm_items* (with audio added) and the list of
             extracted audio items.
         """
-        video_items = self.data_parser.parse_mm_data({"video": mm_data["video"]})
-        videos = video_items.get_items("video", VideoProcessorItems)
+        videos = mm_items.get_items("video", VideoProcessorItems)
         assert isinstance(videos.metadata, list)
         metadata_list = videos.metadata
 
@@ -1424,37 +1423,42 @@ def _extract_audio_from_videos(
             )
             del metadata["original_video_bytes"]
 
-        mm_data = dict(mm_data)
-        mm_data["audio"] = audio_items
-        return mm_data, audio_items
+        audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items})
+        mm_items = MultiModalDataItems({**mm_items, **audio_parsed})
+        return mm_items, audio_items
 
     def apply(
         self,
         prompt: str | list[int],
-        mm_data: MultiModalDataDict,
-        hf_processor_mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        mm_uuid_items: MultiModalUUIDItems | None = None,
+        hf_processor_mm_kwargs: Mapping[str, object] | None = None,
         tokenization_kwargs: Mapping[str, object] | None = None,
-        *,
-        mm_uuids: MultiModalUUIDDict | None = None,
     ) -> MultiModalInputs:
+        if hf_processor_mm_kwargs is None:
+            hf_processor_mm_kwargs = {}
+
         use_audio_in_video = bool(
             hf_processor_mm_kwargs.get("use_audio_in_video", False)
         )
 
         hf_processor_mm_kwargs = {
-            k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
+            k: v for k, v in hf_processor_mm_kwargs.items()
+            if k != "use_audio_in_video"
         }
 
-        if not (use_audio_in_video and "video" in mm_data and "audio" not in mm_data):
+        if not (use_audio_in_video
+                and "video" in mm_items
+                and "audio" not in mm_items):
             return super().apply(
                 prompt,
-                mm_data,
+                mm_items,
+                mm_uuid_items,
                 hf_processor_mm_kwargs,
                 tokenization_kwargs,
-                mm_uuids=mm_uuids,
             )
 
-        mm_data, audio_items = self._extract_audio_from_videos(mm_data)
+        mm_items, audio_items = self._extract_audio_from_videos(mm_items)
 
         if not isinstance(prompt, str):
             tokenizer = self.info.get_tokenizer()
@@ -1466,8 +1470,6 @@ def apply(
         if tokenization_kwargs is None:
             tokenization_kwargs = {}
 
-        mm_items = self._to_mm_items(mm_data)
-
         # Bypass the cached path: the HF processor must receive the
         # prompt (with injected <so_embedding>) and the audio data
         # together so it can perform audio-token replacement natively.
@@ -1478,9 +1480,9 @@ def apply(
         ) = self._apply_hf_processor(
             prompt,
             mm_items,
+            mm_uuid_items,
             hf_processor_mm_kwargs,
             tokenization_kwargs,
-            mm_uuids=mm_uuids,
         )
 
         prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py
index df1b2b1c1da4..0c078bf2921b 100644
--- a/vllm/multimodal/media/connector.py
+++ b/vllm/multimodal/media/connector.py
@@ -293,7 +293,9 @@ def fetch_video(
         )
 
         video_kwargs = dict(self.media_io_kwargs.get("video", {}))
-        video_kwargs["keep_video_bytes"] = keep_video_bytes
+        video_kwargs["keep_video_bytes"] = keep_video_bytes or video_kwargs.get(
+            "keep_video_bytes", False
+        )
 
         video_io = VideoMediaIO(image_io, **video_kwargs)
 
@@ -320,7 +322,9 @@ async def fetch_video_async(
         )
 
         video_kwargs = dict(self.media_io_kwargs.get("video", {}))
-        video_kwargs["keep_video_bytes"] = keep_video_bytes
+        video_kwargs["keep_video_bytes"] = keep_video_bytes or video_kwargs.get(
+            "keep_video_bytes", False
+        )
 
         video_io = VideoMediaIO(image_io, **video_kwargs)
 

From 7f0c642827e1811342ba700818d080467d8e0c61 Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Wed, 25 Feb 2026 16:35:17 +0100
Subject: [PATCH 23/39] Refactor conditional statements in
 NanoNemotronVLMultiModalProcessor for improved readability. Consolidated
 multi-line conditions into single lines to enhance clarity in the handling of
 audio and video inputs.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 464b3a270e22..526cc82549b9 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1443,13 +1443,10 @@ def apply(
         )
 
         hf_processor_mm_kwargs = {
-            k: v for k, v in hf_processor_mm_kwargs.items()
-            if k != "use_audio_in_video"
+            k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
         }
 
-        if not (use_audio_in_video
-                and "video" in mm_items
-                and "audio" not in mm_items):
+        if not (use_audio_in_video and "video" in mm_items and "audio" not in mm_items):
             return super().apply(
                 prompt,
                 mm_items,

From 28359296ad532a59847e2cdd64e3f1dd3d576036 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Fri, 27 Feb 2026 10:33:11 +0200
Subject: [PATCH 24/39] Update vllm/model_executor/models/nano_nemotron_vl.py

Co-authored-by: Roger Wang <hey@rogerw.io>
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 526cc82549b9..bec4a181e01d 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1359,8 +1359,6 @@ def get_replacement_custom(item_idx: int):
                     image_height=image_size.height,
                     max_num_tiles=max_num_tiles,
                 )
-            msg = f"Determined num tokens of image: {feature_size=}"
-            logger.debug(msg)
 
             num_patches = None
             local_image_num_patches = image_num_patches

From 80e469d27085484407db6dbd33ce5a871697d963 Mon Sep 17 00:00:00 2001
From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Date: Fri, 27 Feb 2026 10:40:52 +0200
Subject: [PATCH 25/39] comment clarifying parakeet.py is a dependency of
 nano_nemotron_vl.py

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/parakeet.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vllm/model_executor/models/parakeet.py b/vllm/model_executor/models/parakeet.py
index beeba9ddf28c..8c5539251c45 100644
--- a/vllm/model_executor/models/parakeet.py
+++ b/vllm/model_executor/models/parakeet.py
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
+"""
+Modules below used for the audio encoder component in: models/nano_nemotron_vl.py
+"""
 
 from collections.abc import Iterable
 from dataclasses import asdict

From 2119871c3b2c8e032fdc6f2f9155fce8c39d4acd Mon Sep 17 00:00:00 2001
From: root <root@pool0-02416.cm.cluster>
Date: Fri, 27 Feb 2026 11:05:15 -0800
Subject: [PATCH 26/39] Enhance error handling for audio extraction and video
 loading in multimodal processing. Added checks for empty video bytes in
 `extract_audio_from_video_bytes` and `NanoNemotronVLMultiModalProcessor`,
 ensuring proper configuration with `keep_video_bytes`. Updated `VideoMediaIO`
 to retain original video bytes based on the new parameter. Added new video
 files for testing.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 39 ++++++++-----------
 1 file changed, 16 insertions(+), 23 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index bec4a181e01d..6d769550776c 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -74,7 +74,7 @@
     MultiModalUUIDItems,
     VideoProcessorItems,
 )
-from vllm.multimodal.processing import BaseDummyInputsBuilder
+from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs, TimingContext
 from vllm.multimodal.processing.processor import (
     BaseMultiModalProcessor,
     BaseProcessingInfo,
@@ -1427,13 +1427,10 @@ def _extract_audio_from_videos(
 
     def apply(
         self,
-        prompt: str | list[int],
-        mm_items: MultiModalDataItems,
-        mm_uuid_items: MultiModalUUIDItems | None = None,
-        hf_processor_mm_kwargs: Mapping[str, object] | None = None,
-        tokenization_kwargs: Mapping[str, object] | None = None,
+        processor_inputs: ProcessorInputs,
+        timing_ctx: TimingContext | None = None,
     ) -> MultiModalInputs:
-        if hf_processor_mm_kwargs is None:
+        if (hf_processor_mm_kwargs := processor_inputs.hf_processor_mm_kwargs) is None:
             hf_processor_mm_kwargs = {}
 
         use_audio_in_video = bool(
@@ -1444,26 +1441,25 @@ def apply(
             k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
         }
 
-        if not (use_audio_in_video and "video" in mm_items and "audio" not in mm_items):
+        processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs    
+
+        if not (use_audio_in_video and "video" in processor_inputs.mm_data_items and "audio" not in processor_inputs.mm_data_items):
             return super().apply(
-                prompt,
-                mm_items,
-                mm_uuid_items,
-                hf_processor_mm_kwargs,
-                tokenization_kwargs,
+                processor_inputs,
+                timing_ctx,
             )
 
-        mm_items, audio_items = self._extract_audio_from_videos(mm_items)
+        mm_items, audio_items = self._extract_audio_from_videos(processor_inputs.mm_data_items)
 
-        if not isinstance(prompt, str):
+        if not isinstance(processor_inputs.prompt, str):
             tokenizer = self.info.get_tokenizer()
-            prompt = tokenizer.decode(prompt, skip_special_tokens=False)
+            prompt = tokenizer.decode(processor_inputs.prompt, skip_special_tokens=False)
 
         for _ in audio_items:
             prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
 
-        if tokenization_kwargs is None:
-            tokenization_kwargs = {}
+        if processor_inputs.tokenization_kwargs is None:
+            processor_inputs.tokenization_kwargs = {}
 
         # Bypass the cached path: the HF processor must receive the
         # prompt (with injected <so_embedding>) and the audio data
@@ -1473,11 +1469,8 @@ def apply(
             mm_info,
             is_update_applied,
         ) = self._apply_hf_processor(
-            prompt,
-            mm_items,
-            mm_uuid_items,
-            hf_processor_mm_kwargs,
-            tokenization_kwargs,
+            processor_inputs,
+            timing_ctx=timing_ctx,
         )
 
         prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(

From 281f6f3c842fcfd28f78a32eeafa3cc9767097b7 Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Fri, 27 Feb 2026 20:07:38 +0100
Subject: [PATCH 27/39] Refactor code style in `nano_nemotron_vl.py` for
 improved readability. Adjusted import statements and formatted conditional
 checks for audio extraction from video, ensuring consistent indentation and
 line breaks.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 6d769550776c..ff65f82714f6 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -71,10 +71,13 @@
     ImageSize,
     MultiModalDataItems,
     MultiModalDataParser,
-    MultiModalUUIDItems,
     VideoProcessorItems,
 )
-from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs, TimingContext
+from vllm.multimodal.processing import (
+    BaseDummyInputsBuilder,
+    ProcessorInputs,
+    TimingContext,
+)
 from vllm.multimodal.processing.processor import (
     BaseMultiModalProcessor,
     BaseProcessingInfo,
@@ -1441,19 +1444,27 @@ def apply(
             k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
         }
 
-        processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs    
+        processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs
 
-        if not (use_audio_in_video and "video" in processor_inputs.mm_data_items and "audio" not in processor_inputs.mm_data_items):
+        if not (
+            use_audio_in_video
+            and "video" in processor_inputs.mm_data_items
+            and "audio" not in processor_inputs.mm_data_items
+        ):
             return super().apply(
                 processor_inputs,
                 timing_ctx,
             )
 
-        mm_items, audio_items = self._extract_audio_from_videos(processor_inputs.mm_data_items)
+        mm_items, audio_items = self._extract_audio_from_videos(
+            processor_inputs.mm_data_items
+        )
 
         if not isinstance(processor_inputs.prompt, str):
             tokenizer = self.info.get_tokenizer()
-            prompt = tokenizer.decode(processor_inputs.prompt, skip_special_tokens=False)
+            prompt = tokenizer.decode(
+                processor_inputs.prompt, skip_special_tokens=False
+            )
 
         for _ in audio_items:
             prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)

From c3e48c6149021bbdab05bd4a171f52a7b8b65f55 Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Sat, 28 Feb 2026 10:39:37 +0100
Subject: [PATCH 28/39] Add NemotronVLVideoBackend for enhanced video loading
 and improve audio extraction error handling

- Introduced `NemotronVLVideoBackend` to handle video loading with an option to retain original video bytes.
- Refactored audio extraction logic in `extract_audio_from_video_bytes` to ensure proper error handling and resource management.
- Updated `NanoNemotronVLMultiModalProcessor` to correctly process and store prompt data after audio extraction.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 .../model_executor/models/nano_nemotron_vl.py | 10 ++++---
 vllm/multimodal/media/audio.py                | 25 +++++++++--------
 vllm/multimodal/media/video.py                |  4 ---
 vllm/multimodal/video.py                      | 27 +++++++++++++++++++
 4 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index ff65f82714f6..94ad00a17c97 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1459,16 +1459,18 @@ def apply(
         mm_items, audio_items = self._extract_audio_from_videos(
             processor_inputs.mm_data_items
         )
+        processor_inputs.mm_data_items = mm_items
 
-        if not isinstance(processor_inputs.prompt, str):
+        prompt = processor_inputs.prompt
+        if not isinstance(prompt, str):
             tokenizer = self.info.get_tokenizer()
-            prompt = tokenizer.decode(
-                processor_inputs.prompt, skip_special_tokens=False
-            )
+            prompt = tokenizer.decode(prompt, skip_special_tokens=False)
 
         for _ in audio_items:
             prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
 
+        processor_inputs.prompt = prompt
+
         if processor_inputs.tokenization_kwargs is None:
             processor_inputs.tokenization_kwargs = {}
 
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index e6dcbb7ac30c..c8b24fdf8cb8 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -42,22 +42,25 @@ def extract_audio_from_video_bytes(
             "audio-in-video extraction."
         )
     try:
-        container = av.open(BytesIO(data))
+        with av.open(BytesIO(data)) as container:
+            if not container.streams.audio:
+                raise ValueError("No audio stream found in the video.")
+            stream = container.streams.audio[0]
+            native_sr = stream.rate
+
+            chunks: list[npt.NDArray] = []
+            for frame in container.decode(audio=0):
+                # to_ndarray() returns shape (channels, samples) for planar
+                # formats and (1, samples) for packed formats.
+                arr = frame.to_ndarray()
+                chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
+    except ValueError:
+        raise
     except Exception as e:
         raise ValueError(
             "Invalid or corrupted video data when extracting audio. "
             "Ensure the input is valid video bytes (e.g. a complete MP4)."
         ) from e
-    stream = container.streams.audio[0]
-    native_sr = stream.rate
-
-    chunks: list[npt.NDArray] = []
-    for frame in container.decode(audio=0):
-        # to_ndarray() returns shape (channels, samples) for planar
-        # formats and (1, samples) for packed formats.
-        arr = frame.to_ndarray()
-        chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
-    container.close()
 
     if not chunks:
         raise ValueError("No audio found in the video.")
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index 7a2b8893db74..96e2be6b0744 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -40,7 +40,6 @@ def __init__(
         video_loader_backend = (
             kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
         )
-        self.keep_video_bytes = kwargs.pop("keep_video_bytes", False)
 
         self.kwargs = kwargs
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
@@ -49,9 +48,6 @@ def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
         video, metadata = self.video_loader.load_bytes(
             data, num_frames=self.num_frames, **self.kwargs
         )
-        if self.keep_video_bytes:
-            metadata = dict(metadata)
-            metadata["original_video_bytes"] = data
         return video, metadata
 
     def load_base64(
diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py
index fb4e19fa6745..e959f11b7a0c 100644
--- a/vllm/multimodal/video.py
+++ b/vllm/multimodal/video.py
@@ -431,6 +431,33 @@ def load_bytes(
         return frames, metadata
 
 
+@VIDEO_LOADER_REGISTRY.register("nemotron_vl")
+class NemotronVLVideoBackend(OpenCVVideoBackend):
+    @classmethod
+    def load_bytes(
+        cls,
+        data: bytes,
+        num_frames: int = -1,
+        fps: int = -1,
+        max_duration: int = 300,
+        frame_recovery: bool = False,
+        keep_video_bytes: bool = False,
+        **kwargs,
+    ) -> tuple[npt.NDArray, dict[str, Any]]:
+        frames, metadata = OpenCVVideoBackend.load_bytes(
+            data,
+            num_frames=num_frames,
+            fps=fps,
+            max_duration=max_duration,
+            frame_recovery=frame_recovery,
+            **kwargs,
+        )
+        if keep_video_bytes:
+            metadata = dict(metadata)
+            metadata["original_video_bytes"] = data
+        return frames, metadata
+
+
 @VIDEO_LOADER_REGISTRY.register("molmo2")
 class Molmo2VideoBackend(VideoLoader):
     def get_cv2_video_api(self):

From 27eb57b0a91b171fac9de7b20df5931d74b5abcf Mon Sep 17 00:00:00 2001
From: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Date: Sun, 1 Mar 2026 18:44:57 -0500
Subject: [PATCH 29/39] [Attention] FA4 integration (#32974)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/v1/attention/backends/fa_utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py
index 4039316c36c4..9658a7e3c175 100644
--- a/vllm/v1/attention/backends/fa_utils.py
+++ b/vllm/v1/attention/backends/fa_utils.py
@@ -55,6 +55,9 @@ def get_scheduler_metadata(*args: Any, **kwargs: Any) -> None:  # type: ignore[m
 def get_flash_attn_version(
     requires_alibi: bool = False, head_size: int | None = None
 ) -> int | None:
+    # import here to avoid circular dependencies
+    from vllm.platforms import current_platform
+
     if current_platform.is_xpu():
         return 2
     if current_platform.is_rocm():

From 140746eb3a6e2d935091e506380f4a2cc0d35b80 Mon Sep 17 00:00:00 2001
From: wangxiyuan <wangxiyuan1007@gmail.com>
Date: Mon, 2 Mar 2026 17:36:54 +0800
Subject: [PATCH 30/39] [Misc] Cleanup useless `current_platform` import
 (#35715)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/v1/attention/backends/fa_utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py
index 9658a7e3c175..4039316c36c4 100644
--- a/vllm/v1/attention/backends/fa_utils.py
+++ b/vllm/v1/attention/backends/fa_utils.py
@@ -55,9 +55,6 @@ def get_scheduler_metadata(*args: Any, **kwargs: Any) -> None:  # type: ignore[m
 def get_flash_attn_version(
     requires_alibi: bool = False, head_size: int | None = None
 ) -> int | None:
-    # import here to avoid circular dependencies
-    from vllm.platforms import current_platform
-
     if current_platform.is_xpu():
         return 2
     if current_platform.is_rocm():

From e3c3e38192a5799cedd499bf035eac469ade5d0e Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster>
Date: Mon, 2 Mar 2026 02:08:41 -0800
Subject: [PATCH 31/39] [Feat] Add NemotronHNanoVLV2 model configuration and
 update video metadata handling

- Introduced `NemotronHNanoVLV2Config` class for model configuration with video backend support.
- Updated `NemotronVLVideoBackend` to always include original video bytes in metadata, removing the `keep_video_bytes` parameter.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/config.py | 12 ++++++++++++
 vllm/multimodal/video.py             |  8 ++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py
index 2ec219d40233..8fd6308ba5b6 100644
--- a/vllm/model_executor/models/config.py
+++ b/vllm/model_executor/models/config.py
@@ -648,6 +648,17 @@ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
         model_config.hf_config.embedding_size = model_config.hf_config.num_labels
 
 
+class NemotronHNanoVLV2Config(VerifyAndUpdateConfig):
+    @staticmethod
+    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
+        mm_config = model_config.multimodal_config
+        if mm_config is not None:
+            video_kwargs = mm_config.media_io_kwargs.setdefault(
+                "video", {}
+            )
+            video_kwargs.setdefault("video_backend", "nemotron_vl")
+
+
 MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
     "GteModel": SnowflakeGteNewModelConfig,
     "GteNewModel": GteNewModelConfig,
@@ -675,4 +686,5 @@ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
     "Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
     "Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
     "VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
+    "NemotronH_Nano_VL_V2": NemotronHNanoVLV2Config,
 }
diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py
index e959f11b7a0c..e7a8ec8f86fe 100644
--- a/vllm/multimodal/video.py
+++ b/vllm/multimodal/video.py
@@ -441,7 +441,6 @@ def load_bytes(
         fps: int = -1,
         max_duration: int = 300,
         frame_recovery: bool = False,
-        keep_video_bytes: bool = False,
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         frames, metadata = OpenCVVideoBackend.load_bytes(
@@ -452,9 +451,10 @@ def load_bytes(
             frame_recovery=frame_recovery,
             **kwargs,
         )
-        if keep_video_bytes:
-            metadata = dict(metadata)
-            metadata["original_video_bytes"] = data
+        
+        metadata = dict(metadata)
+        metadata["original_video_bytes"] = data
+
         return frames, metadata
 
 

From 0b377b94fbd87bdaf8ac3db338bd69620a7c442e Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Mon, 2 Mar 2026 11:10:07 +0100
Subject: [PATCH 32/39] Format w/ pre-commit

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/config.py | 4 +---
 vllm/multimodal/video.py             | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py
index 8fd6308ba5b6..c212f351640a 100644
--- a/vllm/model_executor/models/config.py
+++ b/vllm/model_executor/models/config.py
@@ -653,9 +653,7 @@ class NemotronHNanoVLV2Config(VerifyAndUpdateConfig):
     def verify_and_update_model_config(model_config: "ModelConfig") -> None:
         mm_config = model_config.multimodal_config
         if mm_config is not None:
-            video_kwargs = mm_config.media_io_kwargs.setdefault(
-                "video", {}
-            )
+            video_kwargs = mm_config.media_io_kwargs.setdefault("video", {})
             video_kwargs.setdefault("video_backend", "nemotron_vl")
 
 
diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py
index e7a8ec8f86fe..27ba58b74ede 100644
--- a/vllm/multimodal/video.py
+++ b/vllm/multimodal/video.py
@@ -451,7 +451,7 @@ def load_bytes(
             frame_recovery=frame_recovery,
             **kwargs,
         )
-        
+
         metadata = dict(metadata)
         metadata["original_video_bytes"] = data
 

From f3150077df14a8cb312beda9d1a9ea694c1d467b Mon Sep 17 00:00:00 2001
From: root <root@pool0-04880.cm.cluster>
Date: Mon, 2 Mar 2026 03:16:29 -0800
Subject: [PATCH 33/39] [Refactor] Simplify video fetching in
 MultiModalContentParser and MediaConnector

- Removed the `keep_video_bytes` parameter from `fetch_video` and `fetch_video_async` methods.
- Updated the `parse_video` methods to directly call the video fetching functions without the `keep_video_bytes` logic.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py     | 14 ++------------
 vllm/multimodal/media/connector.py | 18 ++----------------
 2 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index b65f859a3659..6db74041be07 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -874,14 +874,7 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
-        keep_bytes = bool(self._tracker._mm_processor_kwargs.get("use_audio_in_video"))
-        video = (
-            self._connector.fetch_video(
-                video_url=video_url, keep_video_bytes=keep_bytes
-            )
-            if video_url
-            else None
-        )
+        video = self._connector.fetch_video(video_url=video_url) if video_url else None
 
         placeholder = self._tracker.add("video", (video, uuid))
         self._add_placeholder("video", placeholder)
@@ -1024,11 +1017,8 @@ def parse_input_audio(
         return self.parse_audio(audio_url, uuid)
 
     async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
-        keep_bytes = bool(self._tracker._mm_processor_kwargs.get("use_audio_in_video"))
         video = (
-            await self._connector.fetch_video_async(
-                video_url=video_url, keep_video_bytes=keep_bytes
-            )
+            await self._connector.fetch_video_async(video_url=video_url)
             if video_url
             else None
         )
diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py
index 0c078bf2921b..784a4ca35abf 100644
--- a/vllm/multimodal/media/connector.py
+++ b/vllm/multimodal/media/connector.py
@@ -283,7 +283,6 @@ def fetch_video(
         video_url: str,
         *,
         image_mode: str = "RGB",
-        keep_video_bytes: bool = False,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         """
         Load video from an HTTP or base64 data URL.
@@ -291,13 +290,7 @@ def fetch_video(
         image_io = ImageMediaIO(
             image_mode=image_mode, **self.media_io_kwargs.get("image", {})
         )
-
-        video_kwargs = dict(self.media_io_kwargs.get("video", {}))
-        video_kwargs["keep_video_bytes"] = keep_video_bytes or video_kwargs.get(
-            "keep_video_bytes", False
-        )
-
-        video_io = VideoMediaIO(image_io, **video_kwargs)
+        video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
 
         return self.load_from_url(
             video_url,
@@ -310,7 +303,6 @@ async def fetch_video_async(
         video_url: str,
         *,
         image_mode: str = "RGB",
-        keep_video_bytes: bool = False,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         """
         Asynchronously load video from an HTTP or base64 data URL.
@@ -320,13 +312,7 @@ async def fetch_video_async(
         image_io = ImageMediaIO(
             image_mode=image_mode, **self.media_io_kwargs.get("image", {})
         )
-
-        video_kwargs = dict(self.media_io_kwargs.get("video", {}))
-        video_kwargs["keep_video_bytes"] = keep_video_bytes or video_kwargs.get(
-            "keep_video_bytes", False
-        )
-
-        video_io = VideoMediaIO(image_io, **video_kwargs)
+        video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
 
         return await self.load_from_url_async(
             video_url,

From 90e60307f1e0c4394f45b0cf68dec1130f1c6c70 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@nvidia.com>
Date: Mon, 2 Mar 2026 13:45:38 +0100
Subject: [PATCH 34/39] run pre-commit formatting

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/multimodal/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index fb0cedadb814..886756c99da9 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import mimetypes
-import warnings
 from collections import defaultdict
 from collections.abc import Generator, Sequence
 from itertools import groupby

From 65917a5ee5d8ffacf4b4a0434b3e327c52e5f5a5 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@nvidia.com>
Date: Mon, 2 Mar 2026 14:16:49 +0100
Subject: [PATCH 35/39] [Refactor] Update video metadata handling in
 NanoNemotronVLMultiModalProcessor

- Removed the `original_video_bytes` from metadata to prevent input modification.
- Created new `VideoProcessorItems` and `MultiModalDataItems` to encapsulate updated video and audio items.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/model_executor/models/nano_nemotron_vl.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 94ad00a17c97..1bb9155fa519 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1422,10 +1422,21 @@ def _extract_audio_from_videos(
                     sr=target_sr,
                 )
             )
-            del metadata["original_video_bytes"]
+
+        # Create a new VideoProcessorItems with metadata that does not contain
+        # the large video bytes, to avoid modifying the input `mm_items`.
+        new_metadata_list = [
+            {k: v for k, v in meta.items() if k != "original_video_bytes"}
+            for meta in metadata_list
+        ]
+        new_videos = VideoProcessorItems(data=videos.data, metadata=new_metadata_list)
 
         audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items})
-        mm_items = MultiModalDataItems({**mm_items, **audio_parsed})
+
+        # Create a new MultiModalDataItems with the new video and audio items.
+        new_mm_items_dict = {**mm_items, **audio_parsed, "video": new_videos}
+        mm_items = MultiModalDataItems(new_mm_items_dict)
+
         return mm_items, audio_items
 
     def apply(

From f89b7b1517cb248767ceff86fdda24038c1b5ed5 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@nvidia.com>
Date: Mon, 2 Mar 2026 19:54:56 +0100
Subject: [PATCH 36/39] [Refactor] Simplify MultiModalItemTracker
 initialization and update video loading method

- Removed unnecessary `mm_processor_kwargs` parameter from `MultiModalItemTracker` and its related functions.
- Updated `load_bytes` method in `VideoMediaIO` to streamline return values by eliminating intermediate variable.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py       | 12 ++----------
 vllm/model_executor/models/config.py |  2 +-
 vllm/multimodal/media/video.py       |  3 +--
 3 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index 6db74041be07..ce1f3ce792a7 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -464,12 +464,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
     def __init__(
         self,
         model_config: ModelConfig,
-        mm_processor_kwargs: dict[str, Any] | None = None,
     ):
         super().__init__()
 
         self._model_config = model_config
-        self._mm_processor_kwargs = mm_processor_kwargs or {}
 
         self._items_by_modality = defaultdict[str, list[_T]](list)
         # Track original modality for each vision_chunk item (image or video)
@@ -1529,16 +1527,13 @@ def parse_chat_messages(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
     content_format: ChatTemplateContentFormat,
-    mm_processor_kwargs: dict[str, Any] | None = None,
 ) -> tuple[
     list[ConversationMessage],
     MultiModalDataDict | None,
     MultiModalUUIDDict | None,
 ]:
     conversation: list[ConversationMessage] = []
-    mm_tracker = MultiModalItemTracker(
-        model_config, mm_processor_kwargs=mm_processor_kwargs
-    )
+    mm_tracker = MultiModalItemTracker(model_config)
 
     for msg in messages:
         sub_messages = _parse_chat_message_content(
@@ -1565,16 +1560,13 @@ async def parse_chat_messages_async(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
     content_format: ChatTemplateContentFormat,
-    mm_processor_kwargs: dict[str, Any] | None = None,
 ) -> tuple[
     list[ConversationMessage],
     MultiModalDataDict | None,
     MultiModalUUIDDict | None,
 ]:
     conversation: list[ConversationMessage] = []
-    mm_tracker = AsyncMultiModalItemTracker(
-        model_config, mm_processor_kwargs=mm_processor_kwargs
-    )
+    mm_tracker = AsyncMultiModalItemTracker(model_config)
 
     for msg in messages:
         sub_messages = _parse_chat_message_content(
diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py
index c212f351640a..440285ffd8fe 100644
--- a/vllm/model_executor/models/config.py
+++ b/vllm/model_executor/models/config.py
@@ -662,6 +662,7 @@ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
     "GteNewModel": GteNewModelConfig,
     "GteNewForSequenceClassification": GteNewModelConfig,
     "Gemma3TextModel": Gemma3TextModelConfig,
+    "NemotronH_Nano_VL_V2": NemotronHNanoVLV2Config,
     "LlamaBidirectionalForSequenceClassification": LlamaBidirectionalConfig,
     "LlamaBidirectionalModel": LlamaBidirectionalConfig,
     "LlamaNemotronVLModel": LlamaNemotronVLConfig,
@@ -684,5 +685,4 @@ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
     "Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
     "Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
     "VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
-    "NemotronH_Nano_VL_V2": NemotronHNanoVLV2Config,
 }
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index 96e2be6b0744..7ca5d116bf4b 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -45,10 +45,9 @@ def __init__(
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
 
     def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
-        video, metadata = self.video_loader.load_bytes(
+        return self.video_loader.load_bytes(
             data, num_frames=self.num_frames, **self.kwargs
         )
-        return video, metadata
 
     def load_base64(
         self, media_type: str, data: str

From 5769eb2050d39cef81bfab65bbcf4bc3662b5e14 Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@nvidia.com>
Date: Mon, 2 Mar 2026 20:39:48 +0100
Subject: [PATCH 37/39] undo unnecessary changes

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/entrypoints/chat_utils.py | 9 ++-------
 vllm/multimodal/media/video.py | 1 -
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index ce1f3ce792a7..c48d7bea983c 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -461,10 +461,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
     maximum per prompt.
     """
 
-    def __init__(
-        self,
-        model_config: ModelConfig,
-    ):
+    def __init__(self, model_config: ModelConfig):
         super().__init__()
 
         self._model_config = model_config
@@ -1016,9 +1013,7 @@ def parse_input_audio(
 
     async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
         video = (
-            await self._connector.fetch_video_async(video_url=video_url)
-            if video_url
-            else None
+            await self._connector.fetch_video_async(video_url) if video_url else None
         )
         return video, uuid
 
diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py
index 7ca5d116bf4b..00ce9fc30a6c 100644
--- a/vllm/multimodal/media/video.py
+++ b/vllm/multimodal/media/video.py
@@ -40,7 +40,6 @@ def __init__(
         video_loader_backend = (
             kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
         )
-
         self.kwargs = kwargs
         self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
 

From a53c6f107bde82470694b23f888fb15314f3a7d4 Mon Sep 17 00:00:00 2001
From: Andrii <askliar@nvidia.com>
Date: Tue, 3 Mar 2026 05:36:54 -0800
Subject: [PATCH 38/39] [Refactor] Revise NemotronHNanoVLV2Config and update
 video backend handling

- Reintroduced `NemotronHNanoVLV2Config` class for model configuration with video backend support.
- Refactored `NemotronVLVideoBackend` to ensure original video bytes are included in metadata.
- Simplified audio extraction in `NanoNemotronVLMultiModalProcessor` by removing unnecessary parameters and improving prompt handling.

Signed-off-by: Andrii Skliar <askliar@nvidia.com>

Signed-off-by: Andrii <askliar@nvidia.com>
---
 vllm/model_executor/models/config.py          | 18 +++----
 .../model_executor/models/nano_nemotron_vl.py | 15 ++----
 vllm/multimodal/media/audio.py                | 15 ++----
 vllm/multimodal/video.py                      | 54 +++++++++----------
 4 files changed, 44 insertions(+), 58 deletions(-)

diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py
index 440285ffd8fe..29ad66e9c4b5 100644
--- a/vllm/model_executor/models/config.py
+++ b/vllm/model_executor/models/config.py
@@ -614,6 +614,15 @@ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
             cache_config.mamba_ssm_cache_dtype = mamba_ssm_cache_dtype
 
 
+class NemotronHNanoVLV2Config(VerifyAndUpdateConfig):
+    @staticmethod
+    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
+        mm_config = model_config.multimodal_config
+        if mm_config is not None:
+            video_kwargs = mm_config.media_io_kwargs.setdefault("video", {})
+            video_kwargs.setdefault("video_backend", "nemotron_vl")
+
+
 class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
     @staticmethod
     def verify_and_update_config(vllm_config: "VllmConfig") -> None:
@@ -648,15 +657,6 @@ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
         model_config.hf_config.embedding_size = model_config.hf_config.num_labels
 
 
-class NemotronHNanoVLV2Config(VerifyAndUpdateConfig):
-    @staticmethod
-    def verify_and_update_model_config(model_config: "ModelConfig") -> None:
-        mm_config = model_config.multimodal_config
-        if mm_config is not None:
-            video_kwargs = mm_config.media_io_kwargs.setdefault("video", {})
-            video_kwargs.setdefault("video_backend", "nemotron_vl")
-
-
 MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
     "GteModel": SnowflakeGteNewModelConfig,
     "GteNewModel": GteNewModelConfig,
diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py
index 1bb9155fa519..8da88a8f40fc 100644
--- a/vllm/model_executor/models/nano_nemotron_vl.py
+++ b/vllm/model_executor/models/nano_nemotron_vl.py
@@ -1402,10 +1402,6 @@ def _extract_audio_from_videos(
         assert isinstance(videos.metadata, list)
         metadata_list = videos.metadata
 
-        target_sr = None
-        if extractor := self.info.audio_extractor:
-            target_sr = extractor.sampling_rate
-
         audio_items: list[AudioItem] = []
         for metadata in metadata_list:
             video_bytes = metadata.get("original_video_bytes")
@@ -1416,12 +1412,7 @@ def _extract_audio_from_videos(
                     "video must be loaded with keep_video_bytes=True (e.g. via "
                     "the chat API with a model that sets use_audio_in_video)."
                 )
-            audio_items.append(
-                extract_audio_from_video_bytes(
-                    video_bytes,
-                    sr=target_sr,
-                )
-            )
+            audio_items.append(extract_audio_from_video_bytes(video_bytes))
 
         # Create a new VideoProcessorItems with metadata that does not contain
         # the large video bytes, to avoid modifying the input `mm_items`.
@@ -1473,14 +1464,14 @@ def apply(
         processor_inputs.mm_data_items = mm_items
 
         prompt = processor_inputs.prompt
+        tokenizer = self.info.get_tokenizer()
         if not isinstance(prompt, str):
-            tokenizer = self.info.get_tokenizer()
             prompt = tokenizer.decode(prompt, skip_special_tokens=False)
 
         for _ in audio_items:
             prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
 
-        processor_inputs.prompt = prompt
+        processor_inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
 
         if processor_inputs.tokenization_kwargs is None:
             processor_inputs.tokenization_kwargs = {}
diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index c8b24fdf8cb8..838836c048d5 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -19,7 +19,6 @@
 
 def extract_audio_from_video_bytes(
     data: bytes,
-    sr: float | None = None,
 ) -> tuple[npt.NDArray, float]:
     """Extract the audio track from raw video bytes using PyAV.
 
@@ -27,9 +26,12 @@ def extract_audio_from_video_bytes(
     spawned, which is critical to avoid crashing CUDA-active vLLM
     worker processes.
 
+    The returned waveform is at the native sample rate of the video's
+    audio stream.  Resampling to a model-specific rate is left to the
+    downstream :class:`AudioResampler` in the parsing pipeline.
+
     Args:
         data: Raw video file bytes (e.g. from an mp4 file).
-        sr: Target sampling rate.  If ``None``, the native rate is used.
 
     Returns:
         A tuple of ``(waveform, sample_rate)`` suitable for use as an
@@ -50,8 +52,6 @@ def extract_audio_from_video_bytes(
 
             chunks: list[npt.NDArray] = []
             for frame in container.decode(audio=0):
-                # to_ndarray() returns shape (channels, samples) for planar
-                # formats and (1, samples) for packed formats.
                 arr = frame.to_ndarray()
                 chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
     except ValueError:
@@ -64,13 +64,8 @@ def extract_audio_from_video_bytes(
 
     if not chunks:
         raise ValueError("No audio found in the video.")
-    else:
-        audio = np.concatenate(chunks).astype(np.float32)
-
-    if sr is not None and sr != native_sr:
-        audio = librosa.resample(audio, orig_sr=float(native_sr), target_sr=float(sr))
-        return audio, float(sr)
 
+    audio = np.concatenate(chunks).astype(np.float32)
     return audio, float(native_sr)
 
 
diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py
index 27ba58b74ede..bafdfbbbbfe9 100644
--- a/vllm/multimodal/video.py
+++ b/vllm/multimodal/video.py
@@ -431,33 +431,6 @@ def load_bytes(
         return frames, metadata
 
 
-@VIDEO_LOADER_REGISTRY.register("nemotron_vl")
-class NemotronVLVideoBackend(OpenCVVideoBackend):
-    @classmethod
-    def load_bytes(
-        cls,
-        data: bytes,
-        num_frames: int = -1,
-        fps: int = -1,
-        max_duration: int = 300,
-        frame_recovery: bool = False,
-        **kwargs,
-    ) -> tuple[npt.NDArray, dict[str, Any]]:
-        frames, metadata = OpenCVVideoBackend.load_bytes(
-            data,
-            num_frames=num_frames,
-            fps=fps,
-            max_duration=max_duration,
-            frame_recovery=frame_recovery,
-            **kwargs,
-        )
-
-        metadata = dict(metadata)
-        metadata["original_video_bytes"] = data
-
-        return frames, metadata
-
-
 @VIDEO_LOADER_REGISTRY.register("molmo2")
 class Molmo2VideoBackend(VideoLoader):
     def get_cv2_video_api(self):
@@ -776,6 +749,33 @@ def load_bytes(
         return out
 
 
+@VIDEO_LOADER_REGISTRY.register("nemotron_vl")
+class NemotronVLVideoBackend(OpenCVVideoBackend):
+    @classmethod
+    def load_bytes(
+        cls,
+        data: bytes,
+        num_frames: int = -1,
+        fps: int = -1,
+        max_duration: int = 300,
+        frame_recovery: bool = False,
+        **kwargs,
+    ) -> tuple[npt.NDArray, dict[str, Any]]:
+        frames, metadata = OpenCVVideoBackend.load_bytes(
+            data,
+            num_frames=num_frames,
+            fps=fps,
+            max_duration=max_duration,
+            frame_recovery=frame_recovery,
+            **kwargs,
+        )
+
+        metadata = dict(metadata)
+        metadata["original_video_bytes"] = data
+
+        return frames, metadata
+
+
 @VIDEO_LOADER_REGISTRY.register("openpangu")
 class OpenCVDynamicOpenPanguVideoBackend(OpenCVVideoBackend):
     @classmethod

From 06f62d7406e893c0bd6159f8abc623068956858a Mon Sep 17 00:00:00 2001
From: Andrii Skliar <askliar@nvidia.com>
Date: Tue, 3 Mar 2026 15:30:10 +0100
Subject: [PATCH 39/39] Optimize imports

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
---
 vllm/multimodal/media/audio.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py
index 838836c048d5..7f23272151a0 100644
--- a/vllm/multimodal/media/audio.py
+++ b/vllm/multimodal/media/audio.py
@@ -4,18 +4,31 @@
 from io import BytesIO
 from pathlib import Path
 
-import av
-import librosa
 import numpy as np
 import numpy.typing as npt
 import pybase64
-import soundfile
 import torch
 
+from vllm.utils.import_utils import PlaceholderModule
 from vllm.utils.serial_utils import tensor2base64
 
 from .base import MediaIO
 
+try:
+    import librosa
+except ImportError:
+    librosa = PlaceholderModule("librosa")  # type: ignore[assignment]
+
+try:
+    import soundfile
+except ImportError:
+    soundfile = PlaceholderModule("soundfile")  # type: ignore[assignment]
+
+try:
+    import av
+except ImportError:
+    av = PlaceholderModule("av")  # type: ignore[assignment]
+
 
 def extract_audio_from_video_bytes(
     data: bytes,