NVIDIA-AI-Blueprints · cdgamarose-nv · Aug 21, 2026 · Aug 19, 2026 · Aug 19, 2026 · Aug 19, 2026
diff --git a/.agents/skills/aiq-configure-workflow/SKILL.md b/.agents/skills/aiq-configure-workflow/SKILL.md
@@ -1,6 +1,6 @@
 ---
 name: aiq-configure-workflow
-description: Use when composing, adapting, or validating an AI-Q workflow YAML under configs/ — selecting a shipped profile, enabling tools and data_source_registry sources, wiring agents and the chat_deepresearcher_agent workflow, configuring general.telemetry (Phoenix, LangSmith, Weave, OTEL) and general.front_end aiq_api settings, and pre-flighting cross-references before deploy or serve. Hand off deploy to aiq-deploy, live research to aiq-research, prompt/model edits to aiq-customize-prompts-models, and new source code to aiq-add-tool or aiq-add-data-source.
+description: Use when composing, adapting, or validating an AI-Q workflow YAML under configs/ — selecting a shipped profile, enabling tools and data_source_registry sources, wiring agents and the chat_deepresearcher_agent workflow, configuring NeMo Relay observability and general.front_end aiq_api settings, and pre-flighting cross-references before deploy or serve. Hand off deploy to aiq-deploy, live research to aiq-research, prompt/model edits to aiq-customize-prompts-models, and new source code to aiq-add-tool or aiq-add-data-source.
 license: Apache-2.0
 compatibility: Claude Code, Codex, Cursor, OpenCode, and Agent Skills-compatible tools.
 metadata:

diff --git a/.agents/skills/aiq-configure-workflow/references/composing-config.md b/.agents/skills/aiq-configure-workflow/references/composing-config.md
@@ -54,18 +54,14 @@ general:
         level: INFO   # DEBUG | INFO | WARNING | ERROR
 ```
 
-**Tracing** — enable under `general.telemetry.tracing` (exporters can coexist).
-Uncomment the matching block in any `config_web_*.yml` or copy from
-`docs/source/deployment/observability.md`:
+**Observability** — configure NeMo Relay under `workflow.relay`. Relay logging,
+ATOF, and redaction are enabled by default. OTEL is opt-in; uncomment the Relay
+OpenInference endpoint in a default config to send traces to Phoenix. Omit
+`workflow.relay.pricing` unless the workflow intentionally loads an audited
+catalog. See `docs/source/deployment/observability.md`.
 
-| Backend | YAML `_type` | Setup notes |
-|---------|--------------|-------------|
-| Phoenix | `phoenix` | `phoenix serve`; set `endpoint`, `project` |
-| LangSmith | `langsmith` or env-only | `LANGCHAIN_TRACING_V2`, `LANGCHAIN_API_KEY`, `LANGCHAIN_PROJECT` |
-| Weave | `weave` | `WANDB_API_KEY`; `project`, optional `redact_pii` |
-| OpenTelemetry | `otelcollector_redaction` | `endpoint`; redaction + batch fields |
-
-`verbose: true` on `workflow:` or agents adds console detail without a tracer.
+`workflow.relay.logging` controls the console subscriber; agent and workflow
+configs do not have separate verbose switches.
 
 ### `front_end` (`aiq_api`)
 
@@ -165,8 +161,8 @@ and feature guides under `docs/source/customization/`.
 
 | `_type` | Key options to tune | Doc anchor |
 |---------|---------------------|------------|
-| `intent_classifier` | `llm`, `tools`, `llm_timeout`, `verbose` | `configuration-reference.md` § `intent_classifier` |
-| `clarifier_agent` | `llm`, `max_turns`, `exclude_tools`, `verbose` | § `clarifier_agent` |
+| `intent_classifier` | `llm`, `tools`, `llm_timeout` | `configuration-reference.md` § `intent_classifier` |
+| `clarifier_agent` | `llm`, `max_turns`, `exclude_tools` | § `clarifier_agent` |
 | `shallow_research_agent` | `llm`, `max_llm_turns`, `max_tool_iterations`, `exclude_tools` | § `shallow_research_agent` |
 | `deep_research_agent` | role LLMs, `exclude_tools`, `enable_source_router`, `domain_catalog_path`, `enable_citation_verification`, `skills`, `sandbox`, concurrency caps | § `deep_research_agent` |
 
@@ -191,7 +187,6 @@ workflow:
   use_async_deep_research: true   # needs general.front_end
   max_history: 20
   checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db}
-  verbose: true
 ```
 
 Full defaults table: `configuration-reference.md` § `workflow`.

diff --git a/.agents/skills/aiq-configure-workflow/references/config-schema.md b/.agents/skills/aiq-configure-workflow/references/config-schema.md
@@ -26,8 +26,6 @@ uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py <
 - No `data_source_registry`
 - `requires_auth: true` on a source (confirm MCP/OAuth wiring)
 - `use_async_deep_research: true` without `general.front_end`
-- LangSmith tracing without `LANGCHAIN_API_KEY`
-- Weave tracing without `WANDB_API_KEY`
 
 ## Env checklist
 

diff --git a/.agents/skills/aiq-configure-workflow/references/env-vars.md b/.agents/skills/aiq-configure-workflow/references/env-vars.md
@@ -31,15 +31,13 @@ Canonical references:
 | `SEARCHAPI_API_KEY` | SearchAPI paper search |
 | `RAG_SERVER_URL`, `RAG_INGEST_URL` | Foundational RAG profiles |
 
-## Web API, auth, and tracing
+## Web API, auth, and Relay correlation
 
 | Variable | When needed |
 |----------|-------------|
 | `REQUIRE_AUTH` | Enforce API authentication. Requires validator registration. |
-| `AIQ_TRACE_USER_IDENTITY_MODE`, `AIQ_TRACE_USER_IDENTITY_HMAC_SECRET` | User identity tagging for NAT spans. |
-| `AIQ_TRACE_CLIENT_ID_MODE`, `AIQ_TRACE_CLIENT_ID_HMAC_SECRET`, `AIQ_TRACE_CLIENT_IP_HEADERS` | Client tagging for NAT spans. |
-| `LANGCHAIN_TRACING_V2`, `LANGCHAIN_API_KEY`, `LANGCHAIN_PROJECT` | LangSmith tracing. |
-| `WANDB_API_KEY` | Weave tracing. |
+| `AIQ_TRACE_USER_IDENTITY_MODE`, `AIQ_TRACE_USER_IDENTITY_HMAC_SECRET` | User identity tagging for Relay-exported spans. |
+| `AIQ_TRACE_CLIENT_ID_MODE`, `AIQ_TRACE_CLIENT_ID_HMAC_SECRET`, `AIQ_TRACE_CLIENT_IP_HEADERS` | Client tagging for Relay-exported spans. |
 
 ## Sandbox and artifact storage
 

diff --git a/.gitignore b/.gitignore
@@ -44,6 +44,10 @@ env/
 *.log
 logs/
 
+# NeMo Relay local observability output (regeneratable; never commit payloads)
+/.nemo-relay/
+/relay/
+
 # Testing
 .pytest_cache/
 .coverage

diff --git a/.secrets.baseline b/.secrets.baseline
diff --git a/README.md b/README.md
@@ -72,7 +72,7 @@ The NVIDIA AI-Q Blueprint is a deployable research backend built on the [NVIDIA
 - **Expanded sources** — Paper search supports Serper, SerpAPI, and SearchAPI; You.com adds web, contents, general-research, and finance-research tools; Nimble adds configurable web search; focused profiles demonstrate DuckDuckGo news, Polymarket, OpenSearch, and Azure AI Search knowledge retrieval.
 - **Production API and auth** — REST endpoints, async job ownership, per-user OAuth-protected MCP sources, token validator entry points, and provider lifecycle hooks support authenticated deployments; a separate public MCP server exposes stateless research tools for trusted networks.
 - **Opt-in policy controls** — NeMo Guardrails middleware covers selected workflow and agent boundaries, and narrow application-level encryption can protect final async output plus selected artifact-event content.
-- **Observability, profiling, and cost analysis** — NAT-exported async traces preserve task, named-agent, and model/tool hierarchy across concurrent researchers. Tokenomics reports combine profiler traces with pricing configuration for cost, latency, and cache analysis.
+- **Observability, profiling, and cost analysis** — NeMo Relay preserves task, named-agent, LLM, and tool hierarchy across interactive turns and async researchers. ATOF feeds local debugging and tokenomics reports; OTEL exports traces to external observability backends.
 - **Evaluation harnesses** — Built-in benchmarks (for example, FreshQA, DeepResearch) and evaluation scripts to measure quality and iterate on prompts and agent architecture.
 - **Frontend options** — Run through CLI, web UI, or async jobs. Refer to [Getting started](#getting-started) and [Ways to run the agents](#ways-to-run-the-agents).
 - **Deployment options** - Deployment assets for [Docker Compose](deploy/compose/) and [Helm](deploy/helm/deployment-k8s/); the repository source chart honors the Helm release namespace for every namespaced resource.
@@ -478,7 +478,7 @@ For development, contribution, and documentation, refer to:
 - **[Knowledge Layer Setup](sources/knowledge_layer/KNOWLEDGE-LAYER-SETUP.md)**: RAG backends and document ingestion
 - **[Agent Skills](docs/source/integration/agent-skills.md)**: Install the portable AI-Q research skill in compatible coding harnesses
 - **[Skills and Sandbox Example](docs/source/examples/skills-sandbox/index.md)**: Run deep research with built-in skills and Modal sandbox execution
-- **[Profiling and Cost Analysis](docs/source/profiling/index.md)**: Generate tokenomics and latency reports from NAT profiler traces
+- **[Profiling and Cost Analysis](docs/source/profiling/index.md)**: Generate tokenomics and latency reports from Relay ATOF traces
 - **[Docs index](docs/README.md)**: Full documentation list and component docs
 - **[Changelog](docs/source/resources/changelog.md)**: Version history and changes
 

diff --git a/configs/config_cli_default.yml b/configs/config_cli_default.yml
@@ -10,10 +10,6 @@ general:
       console:
         _type: console
         level: INFO
-    # tracing:
-    #   langsmith: # Optional: LangSmith tracing - requires langsmith API key. Set using `export LANGSMITH_API_KEY=<your-langsmith-api-key>`
-    #     _type: langsmith
-    #     project: nvidia-aiq
 
 llms:
   nemotron_lightning_intent_llm:
@@ -133,7 +129,6 @@ functions:
     # exclude_tools: []
     max_turns: 3
     log_response_max_chars: 2000
-    verbose: true
 
   shallow_research_agent:
     _type: shallow_research_agent
@@ -161,3 +156,15 @@ workflow:
   enable_escalation: true
   enable_clarifier: true
   checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db}
+  # Uncomment to export Relay traces to a local Phoenix instance.
+  # relay:
+  #   observability:
+  #     opentelemetry:
+  #       enabled: true
+  #       endpoints:
+  #         - type: openinference
+  #           endpoint: ${RELAY_OTEL_ENDPOINT:-http://localhost:6006/v1/traces}
+  #           service_name: aiq-relay
+  #           resource_attributes:
+  #             openinference.project.name: aiq-relay
+  #             deployment.environment: development
diff --git a/configs/config_domain_routing_and_skills.yml b/configs/config_domain_routing_and_skills.yml
@@ -10,10 +10,6 @@ general:
       console:
         _type: console
         level: INFO
-    # tracing:
-    #   langsmith: # Optional: LangSmith tracing - requires langsmith API key. Set using `export LANGSMITH_API_KEY=<your-langsmith-api-key>`
-    #     _type: langsmith
-    #     project: nvidia-aiq
 
   front_end:
     _type: aiq_api
@@ -220,12 +216,10 @@ functions:
     # tools: omitted -> inherits all from data_source_registry
     exclude_tools:                     # Remove basic variant; deep uses advanced_web_search_tool
       - web_search_tool
-    verbose: true
     domain_catalog_path: configs/domain_catalogs/deep_research_domain_catalog.yml
     skills: deep_research_skills
     sandbox: deep_research_sandbox
 
 workflow:
   _type: deep_research_workflow
-  verbose: true
   use_async_deep_research: true
diff --git a/configs/config_frontier_models.yml b/configs/config_frontier_models.yml
@@ -14,10 +14,6 @@ general:
       console:
         _type: console
         level: INFO
-    # tracing:
-    #   langsmith: # Optional: LangSmith tracing - requires langsmith API key. Set using `export LANGSMITH_API_KEY=<your-langsmith-api-key>`
-    #     _type: langsmith
-    #     project: nvidia-aiq
 
   front_end:
     _type: aiq_api
@@ -132,7 +128,6 @@ functions:
   intent_classifier:
     _type: intent_classifier
     llm: gpt_luna_intent_llm
-    verbose: true
     tools:
       - web_search_tool
       # - paper_search_tool  # Uncomment if SERPER_API_KEY is set
@@ -146,12 +141,10 @@ functions:
       - knowledge_search
     max_turns: 3
     log_response_max_chars: 2000
-    verbose: true
 
   shallow_research_agent:
     _type: shallow_research_agent
     llm: gpt_luna_shallow_llm
-    verbose: true
     tools:
       - web_search_tool
       - knowledge_search
@@ -166,7 +159,6 @@ functions:
     researcher_llm: gpt_luna_agent_llm
     planner_llm: gpt_sol_agent_llm
     writer_llm: gpt_sol_writer_llm
-    verbose: true
     tools:
       # - paper_search_tool  # Uncomment if SERPER_API_KEY is set
       - advanced_web_search_tool
@@ -175,7 +167,6 @@ functions:
 
 workflow:
   _type: chat_deepresearcher_agent
-  verbose: true
   enable_escalation: true
   enable_clarifier: true
   use_async_deep_research: true

diff --git a/configs/config_mcp.yml b/configs/config_mcp.yml
@@ -109,7 +109,6 @@ functions:
     llm: nemotron_ultra_llm
     max_turns: 3
     log_response_max_chars: 2000
-    verbose: true
 
   shallow_research_agent:
     _type: shallow_research_agent

diff --git a/configs/config_openshell.yml b/configs/config_openshell.yml
@@ -127,21 +127,18 @@ functions:
   intent_classifier:
     _type: intent_classifier
     llm: nemotron_lightning_intent_llm
-    verbose: true
 
   clarifier_agent:
     _type: clarifier_agent
     llm: nemotron_ultra_llm
     max_turns: 3
     log_response_max_chars: 2000
-    verbose: true
 
   shallow_research_agent:
     _type: shallow_research_agent
     llm: nemotron_lightning_agent_llm
     exclude_tools:
       - advanced_web_search_tool
-    verbose: true
     max_llm_turns: 10
     max_tool_iterations: 5
 
@@ -198,13 +195,11 @@ functions:
     writer_llm: nemotron_ultra_writer_llm
     exclude_tools:
       - web_search_tool
-    verbose: true
     skills: deep_research_skills
     sandbox: deep_research_sandbox
 
 workflow:
   _type: chat_deepresearcher_agent
-  verbose: true
   enable_escalation: true
   enable_clarifier: true
   use_async_deep_research: true

diff --git a/configs/config_web_azure_ai_search.yml b/configs/config_web_azure_ai_search.yml
@@ -10,10 +10,6 @@ general:
       console:
         _type: console
         level: INFO
-    # tracing:
-    #   langsmith: # Optional: LangSmith tracing - requires langsmith API key. Set using `export LANGSMITH_API_KEY=<your-langsmith-api-key>`
-    #     _type: langsmith
-    #     project: nvidia-aiq
 
   front_end:
     _type: aiq_api
@@ -196,7 +192,6 @@ functions:
     llm: nemotron_lightning_intent_llm
     # tools: omitted -> inherits all from data_source_registry
     # exclude_tools: []
-    verbose: true
 
   clarifier_agent:
     _type: clarifier_agent
@@ -207,15 +202,13 @@ functions:
     max_turns: 3
     enable_plan_approval: true
     log_response_max_chars: 2000
-    verbose: true
 
   shallow_research_agent:
     _type: shallow_research_agent
     llm: nemotron_lightning_agent_llm
     # tools: omitted -> inherits all from data_source_registry
     exclude_tools:                     # Remove advanced variant; shallow uses web_search_tool
       - advanced_web_search_tool
-    verbose: true
     max_llm_turns: 10
     max_tool_iterations: 5
 
@@ -230,11 +223,9 @@ functions:
     # tools: omitted -> inherits all from data_source_registry
     exclude_tools:                     # Remove basic variant; deep uses advanced_web_search_tool
       - web_search_tool
-    verbose: true
 
 workflow:
   _type: chat_deepresearcher_agent
-  verbose: true
   enable_escalation: true
   enable_clarifier: true
   use_async_deep_research: true

diff --git a/configs/config_web_default_guardrails.yml b/configs/config_web_default_guardrails.yml
@@ -232,21 +232,18 @@ functions:
   intent_classifier:
     _type: intent_classifier
     llm: nemotron_lightning_intent_llm
-    verbose: true
 
   clarifier_agent:
     _type: clarifier_agent
     llm: nemotron_ultra_llm
     max_turns: 3
     log_response_max_chars: 2000
-    verbose: true
 
   shallow_research_agent:
     _type: shallow_research_agent
     llm: nemotron_lightning_agent_llm
     exclude_tools:
       - advanced_web_search_tool
-    verbose: true
     max_llm_turns: 10
     max_tool_iterations: 5
 
@@ -259,11 +256,9 @@ functions:
     writer_llm: nemotron_ultra_writer_llm
     exclude_tools:
       - web_search_tool
-    verbose: true
 
 workflow:
   _type: chat_deepresearcher_agent
-  verbose: true
   enable_escalation: true
   enable_clarifier: true
   use_async_deep_research: true