refactor(simple-vlm): separate agentic and voice vision paths

DEPENDENCIES.md

@@ -554,16 +554,21 @@ the latest video frame via streaming VLM and replies with both
 | Sub-project | Package | Internal deps | External deps |
 |---|---|---|---|
 | Orchestrator | `simple-vlm-example` | `xr-ai-launcher` | — |
-| Worker | `simple-vlm-example-worker` | `xr-ai-hub-client [editable]`, `xr-ai-logging [editable]`, `xr-ai-models [editable]`, `xr-ai-nat[vision,voice] [editable]`, `xr-ai-voice [editable]`, `xr-ai-voicegate [editable]` | loguru >=0.7, pyyaml >=6.0 (`xr-ai-voice` pulls in VAD, pipecat-ai, numpy, and scipy; `xr-ai-nat[vision]` pulls in httpx, numpy, and Pillow) |
-
-The packaged worker registers `StreamingVisionConfig` in-process and maps it to
-`VoiceSession` with `xr_ai_nat.adapters.as_voice_handler`. `VoiceSession` owns
-readiness, hub transport, signals, the private Pipecat pipeline, and cleanup;
-`TextMessageInput` routes `"ping"` and ad-hoc text through the same
-participant-aware path as speech. Voice-gate behavior (magic phrases, follow-up
-grace, listening chime, stop acknowledgement), VAD/STT, and sentence-batched TTS
-remain provided by the shared voice runtime. The sample has no direct
-`xr-ai-pipecat` or MCP dependency.
+| Worker | `simple-vlm-example-worker` | `xr-ai-hub-client [editable]`, `xr-ai-logging [editable]`, `xr-ai-models [editable]`, `xr-ai-nat[relay,live-vision] [editable]`, `xr-ai-voice [editable]`, `xr-ai-voicegate [editable]` | loguru >=0.7, pyyaml >=6.0 (`xr-ai-voice` pulls in VAD, pipecat-ai, numpy, and scipy; `xr-ai-nat[live-vision]` pulls in numpy and Pillow) |
+
+The packaged worker constructs the finite `LiveVisionTool` used by agentic
+flows, then maps its separate direct-voice `LiveVisionResponder` to
+`VoiceSession`. They share current-frame acquisition through
+`xr-ai-hub-client`; only the direct voice response uses NeMo Relay's managed
+streaming LLM path under an Agent scope. Camera bytes are redacted from Relay
+telemetry while the provider receives the original frame.
+`VoiceSession` owns readiness, hub transport, signals, the
+private Pipecat pipeline, and cleanup; `TextMessageInput` routes `"ping"` and
+ad-hoc text through the same participant-aware path as speech. Voice-gate
+behavior (magic phrases, follow-up grace, listening chime, stop acknowledgement),
+VAD/STT, and sentence-batched TTS remain provided by the shared voice runtime.
+The sample has no direct `xr-ai-pipecat` or MCP dependency and selects no legacy
+NeMo Agent Toolkit extra.
 
 Worker calls stt-server (8103), vlm-server (8100), and piper-tts-server
 (8105) over HTTP via `xr-ai-models` SDK — no model weights loaded

agent-samples/simple-vlm-example/README.md

@@ -18,11 +18,13 @@ The worker is a package under `worker/simple_vlm_example_worker/`:
 - `prompts/system.txt` owns the VLM system prompt.
 
 `VoiceSession` owns STT/TTS/VLM readiness, the hub voice transport, voice-gate
-processing, streaming TTS, signals, and cleanup. The application registers
-`StreamingVisionConfig` in-process and adapts that native function with
-`xr_ai_nat.adapters.as_voice_handler`. Typed text uses the same participant-aware
-turn path as speech. Participant leave events release cached live-frame state,
-and a newer turn cancels and interrupts a superseded response.
+processing, streaming TTS, signals, and cleanup. The application constructs a
+finite `LiveVisionTool` and maps its separate `LiveVisionResponder` to voice;
+both share participant frame acquisition while only the direct voice response
+streams through Relay's managed LLM path. The camera frame is redacted from
+Relay telemetry. Typed text uses the same
+participant-aware turn path as speech. Participant leave events release cached
+live-frame state, and a newer turn cancels and interrupts a superseded response.
 
 No MCP client or MCP tool invocation is part of this sample.
 

agent-samples/simple-vlm-example/worker/pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "xr-ai-hub-client",
     "xr-ai-logging",
     "xr-ai-models",
-    "xr-ai-nat[vision,voice]",
+    "xr-ai-nat[relay,live-vision]",
     "xr-ai-voice",
     "xr-ai-voicegate",
     "loguru>=0.7",

agent-samples/simple-vlm-example/worker/simple_vlm_example_worker/app.py

@@ -9,31 +9,29 @@
 from pathlib import Path
 
 from loguru import logger
-from nat.builder.workflow_builder import WorkflowBuilder
-from nat.plugin_api import Function
 from xr_ai_logging import setup_logging
 from xr_ai_models import load_models_config, make_stt, make_tts, make_vlm
-from xr_ai_nat.adapters import as_voice_handler
-from xr_ai_nat.functions.vision import (
-    StreamingVisionConfig,
-    VisionRequest,
-)
+from xr_ai_nat.live_vision import LiveVisionResponder, LiveVisionTool, VisionRequest
 from xr_ai_voice import TextMessageInput, VadConfig, VoiceHandler, VoiceSession
 from xr_ai_voicegate import load_voice_gate_config
 
 from .config import WorkerConfig
 
 
-def _make_vision_handler(vision: Function) -> VoiceHandler:
-    return as_voice_handler(
-        vision,
-        request=lambda turn: VisionRequest(
-            participant_id=turn.participant_id,
-            query=turn.text,
-        ),
-        response=lambda chunk: chunk.text,
-        streaming=True,
-    )
+def _make_vision_handler(vision: LiveVisionResponder) -> VoiceHandler:
+    async def handle(turn):
+        async def response():
+            async for chunk in vision.stream(
+                VisionRequest(
+                    participant_id=turn.participant_id,
+                    query=turn.text,
+                )
+            ):
+                yield chunk.text
+
+        return response()
+
+    return handle
 
 
 def _text_transform(default_prompt: str) -> Callable[[str], str]:
@@ -70,15 +68,15 @@ async def run_app(
         idle_timeout_secs=config.idle_timeout_secs,
     )
 
-    async with session, WorkflowBuilder() as builder:
-        vision_config = StreamingVisionConfig(
+    async with session:
+        vision = LiveVisionTool(
             endpoint=session.transport.endpoint,
             vlm=vlm,
             system_prompt=config.system_prompt,
             frame_max_age_s=config.frame_max_age_s,
             frame_timeout_s=config.frame_timeout_s,
         )
-        vision = await builder.add_function("perception", vision_config)
+        voice = LiveVisionResponder(vision)
         TextMessageInput(
             session=session,
             transform=_text_transform(config.default_prompt),
@@ -87,8 +85,8 @@ async def run_app(
 
         logger.info("simple-vlm-example starting")
         await session.run(
-            _make_vision_handler(vision),
-            on_participant_left=vision_config.release,
+            _make_vision_handler(voice),
+            on_participant_left=vision.release,
             interrupt_on_supersede=True,
         )
         logger.info("simple-vlm-example stopped")

agent-sdk/xr-ai-models/README.md

@@ -150,10 +150,13 @@ class VLMService(Protocol):
     capabilities: Capabilities
     async def ask_image(self, image, question, *, system_prompt="",
                         max_tokens=None, temperature=None,
-                        timeout=None) -> ChatResponse: ...
+                        timeout=None, headers=None) -> ChatResponse: ...
     async def ask_video(self, video, question, *, system_prompt="",
                         max_tokens=None, temperature=None,
-                        timeout=None) -> ChatResponse: ...
+                        timeout=None, headers=None) -> ChatResponse: ...
+    def stream(self, image, question, *, system_prompt="",
+               max_tokens=None, temperature=None,
+               timeout=None, headers=None) -> AsyncIterator[str]: ...
     async def health(self) -> bool: ...
 
 class STTService(Protocol):
@@ -175,9 +178,9 @@ class EmbeddingService(Protocol):
 `reasoning_field` knob normalizes `reasoning_content` (nemotron_v3 parser)
 into the same surface.
 
-`LLMService.chat` and `LLMService.stream` accept optional string-valued
-per-call headers for execution context such as Relay session lineage. The
-model profile remains the authority for credentials: callers cannot supply an
+`LLMService` and `VLMService` request methods accept optional string-valued
+per-call headers for execution context such as Relay session lineage. The model
+profile remains the authority for credentials: callers cannot supply an
 `Authorization` header.
 
 ## Remote / hosted-NIM endpoints

agent-sdk/xr-ai-models/xr_ai_models/_openai_compat.py

@@ -471,10 +471,12 @@ async def ask_image(
         max_tokens: int | None = None,
         temperature: float | None = None,
         timeout: float | None = None,
+        headers: Mapping[str, str] | None = None,
     ) -> ChatResponse:
         return await self._llm.chat(
             self._build_messages(image, question, system_prompt),
             max_tokens=max_tokens, temperature=temperature, timeout=timeout,
+            headers=headers,
         )
 
     def _build_video_messages(
@@ -501,6 +503,7 @@ async def ask_video(
         max_tokens: int | None = None,
         temperature: float | None = None,
         timeout: float | None = None,
+        headers: Mapping[str, str] | None = None,
     ) -> ChatResponse:
         if not self.capabilities.video:
             raise ValueError(
@@ -511,6 +514,7 @@ async def ask_video(
         return await self._llm.chat(
             self._build_video_messages(video, question, system_prompt),
             max_tokens=max_tokens, temperature=temperature, timeout=timeout,
+            headers=headers,
         )
 
     async def stream(
@@ -522,10 +526,12 @@ async def stream(
         max_tokens: int | None = None,
         temperature: float | None = None,
         timeout: float | None = None,
+        headers: Mapping[str, str] | None = None,
     ) -> AsyncIterator[str]:
         async for chunk in self._llm.stream(
             self._build_messages(image, question, system_prompt),
             max_tokens=max_tokens, temperature=temperature, timeout=timeout,
+            headers=headers,
         ):
             yield chunk
 

agent-sdk/xr-ai-models/xr_ai_models/_protocols.py

@@ -143,6 +143,7 @@ async def ask_image(
         max_tokens: int | None = None,
         temperature: float | None = None,
         timeout: float | None = None,
+        headers: Mapping[str, str] | None = None,
     ) -> ChatResponse: pass
 
     async def ask_video(
@@ -154,6 +155,7 @@ async def ask_video(
         max_tokens: int | None = None,
         temperature: float | None = None,
         timeout: float | None = None,
+        headers: Mapping[str, str] | None = None,
     ) -> ChatResponse: pass
 
     def stream(
@@ -165,6 +167,7 @@ def stream(
         max_tokens: int | None = None,
         temperature: float | None = None,
         timeout: float | None = None,
+        headers: Mapping[str, str] | None = None,
     ) -> AsyncIterator[str]: pass
 
     async def health(self) -> bool: pass

agent-sdk/xr-ai-nat/README.md

@@ -60,6 +60,22 @@ custom, Fabric-backed, or framework-backed runner through the same registered
 invocation path. Relay observes model calls inside a tool-backed runner; the
 application never calls an LLM client as a separate control path.
 
+## Live vision tool and direct voice responder
+
+Install `xr-ai-nat[relay,live-vision]` for `LiveVisionTool`. The finite
+`look_at_current_frame` tool acquires a participant's current frame and returns
+one complete `VisionResponse` for agentic planning. `LiveVisionResponder`
+shares that tool's frame source and streams only the direct voice path. Both
+call an injected `VLMService` through Relay's matching managed LLM boundary,
+forward controlled Relay headers, and redact the inline camera frame from
+events while the provider receives the original. `LiveVisionTool.release()`
+clears participant frame state.
+
+Relay's managed tool API accepts completed JSON results, while its managed LLM
+API supports streaming. Agentic vision therefore uses `Tool.execute()` and a
+complete result; direct voice remains an application-owned response stream
+with Relay managing the nested model call.
+
 ## Legacy NAT compatibility
 
 ## Shared value models and the service boundary

agent-sdk/xr-ai-nat/xr_ai_nat/__init__.py

@@ -6,4 +6,10 @@
 from .agent_runner import AgentRunner, as_agent_tool
 from .tools import Tool, ToolInvocationResult, ToolSet
 
-__all__ = ["AgentRunner", "Tool", "ToolInvocationResult", "ToolSet", "as_agent_tool"]
+__all__ = [
+    "AgentRunner",
+    "Tool",
+    "ToolInvocationResult",
+    "ToolSet",
+    "as_agent_tool",
+]

agent-sdk/xr-ai-nat/xr_ai_nat/_pixels.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Convert hub frames into JPEG data URLs accepted by VLM services."""
+
+from __future__ import annotations
+
+import base64
+import io
+
+import numpy as np
+from PIL import Image
+from xr_ai_hub import FrameData, PixelFormat
+
+
+def _yuv_to_rgb(y: np.ndarray, u: np.ndarray, v: np.ndarray) -> Image.Image:
+    y = y.astype(np.float32) - 16.0
+    u = u.astype(np.float32) - 128.0
+    v = v.astype(np.float32) - 128.0
+    red = np.clip(1.164 * y + 1.596 * v, 0, 255)
+    green = np.clip(1.164 * y - 0.392 * u - 0.813 * v, 0, 255)
+    blue = np.clip(1.164 * y + 2.017 * u, 0, 255)
+    return Image.fromarray(np.stack([red, green, blue], axis=-1).astype(np.uint8), "RGB")
+
+
+def frame_to_pil(frame: FrameData) -> Image.Image:
+    """Convert a hub frame to RGB pixels suitable for JPEG encoding."""
+
+    width, height = frame.width, frame.height
+    data = np.frombuffer(frame.data, dtype=np.uint8)
+    if frame.fmt == PixelFormat.RGB24:
+        return Image.fromarray(data.reshape(height, width, 3), "RGB")
+    if frame.fmt == PixelFormat.RGBA:
+        return Image.fromarray(data.reshape(height, width, 4), "RGBA").convert("RGB")
+    if frame.fmt == PixelFormat.BGRA:
+        bgra = data.reshape(height, width, 4)
+        return Image.fromarray(bgra[:, :, [2, 1, 0]], "RGB")
+    y_end = width * height
+    y = data[:y_end].reshape(height, width)
+    if frame.fmt == PixelFormat.I420:
+        uv_size = (width // 2) * (height // 2)
+        u = data[y_end : y_end + uv_size].reshape(height // 2, width // 2).repeat(2, 0).repeat(2, 1)
+        v = data[y_end + uv_size :].reshape(height // 2, width // 2).repeat(2, 0).repeat(2, 1)
+        return _yuv_to_rgb(y, u, v)
+    if frame.fmt == PixelFormat.NV12:
+        uv = data[y_end:].reshape(height // 2, width)
+        return _yuv_to_rgb(y, uv[:, 0::2].repeat(2, 0).repeat(2, 1), uv[:, 1::2].repeat(2, 0).repeat(2, 1))
+    raise ValueError(f"Unsupported pixel format: {frame.fmt!r}")
+
+
+def encode_image(image: Image.Image) -> str:
+    """Encode one image as an OpenAI-compatible JPEG data URL."""
+
+    buffer = io.BytesIO()
+    image.save(buffer, format="JPEG", quality=90)
+    return f"data:image/jpeg;base64,{base64.b64encode(buffer.getvalue()).decode()}"

agent-sdk/xr-ai-nat/xr_ai_nat/_relay.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Shared validation at the NeMo Relay service boundary."""
+
+from __future__ import annotations
+
+
+def headers_from_relay(raw: object) -> dict[str, str]:
+    """Return string-only headers supplied by a Relay request intercept."""
+
+    if not isinstance(raw, dict):
+        raise TypeError("Relay LLM request headers must be an object")
+    headers: dict[str, str] = {}
+    for name, value in raw.items():
+        if not isinstance(name, str) or not isinstance(value, str):
+            raise TypeError("Relay LLM request headers must be strings")
+        headers[name] = value
+    return headers
+
+
+__all__ = ["headers_from_relay"]

agent-sdk/xr-ai-nat/xr_ai_nat/agents.py

@@ -14,6 +14,7 @@
 from nemo_relay.codecs import OpenAIChatCodec
 from xr_ai_models import ChatMessage, ChatResponse, LLMService, ToolCall, ToolDef
 
+from ._relay import headers_from_relay
 from .agent_runner import AgentRunner, as_agent_tool
 from .tools import Tool, ToolSet
 
@@ -137,7 +138,7 @@ async def invoke(request: nemo_relay.LLMRequest) -> dict[str, Any]:
                 temperature=_optional_float(content.get("temperature")),
                 enable_thinking=bool(content.get("enable_thinking", False)),
                 thinking_budget=_optional_int(content.get("thinking_budget")),
-                headers=_headers_from_relay(request.headers),
+                headers=headers_from_relay(request.headers),
             )
             return _response_to_openai(response)
 
@@ -243,17 +244,6 @@ def _tools_from_openai(raw: object) -> list[ToolDef] | None:
     return definitions
 
 
-def _headers_from_relay(raw: object) -> dict[str, str]:
-    if not isinstance(raw, dict):
-        raise TypeError("Relay LLM request headers must be an object")
-    headers: dict[str, str] = {}
-    for name, value in raw.items():
-        if not isinstance(name, str) or not isinstance(value, str):
-            raise TypeError("Relay LLM request headers must be strings")
-        headers[name] = value
-    return headers
-
-
 def _response_to_openai(response: ChatResponse) -> dict[str, Any]:
     message = _message_to_openai(
         ChatMessage(role="assistant", content=response.content, tool_calls=response.tool_calls),