From 356439820a621b5f55a378508125f94a399af842 Mon Sep 17 00:00:00 2001 From: Soumili Nandi Date: Mon, 13 Jul 2026 15:45:49 -0700 Subject: [PATCH 1/2] Add AI-Q workflow configuration skill Signed-off-by: Soumili Nandi --- .agents/skills/README.md | 2 +- .agents/skills/aiq-add-data-source/SKILL.md | 1 + .agents/skills/aiq-add-tool/SKILL.md | 1 + .../skills/aiq-configure-workflow/SKILL.md | 89 ++++++ .../assets/config-scaffold.yml | 52 +++ .../references/composing-config.md | 207 ++++++++++++ .../references/config-profiles.md | 26 ++ .../references/config-schema.md | 46 +++ .../references/env-vars.md | 57 ++++ .../scripts/validate_config.py | 301 ++++++++++++++++++ .../aiq-customize-prompts-models/SKILL.md | 1 + .claude/skills/aiq-configure-workflow | 1 + 12 files changed, 783 insertions(+), 1 deletion(-) create mode 100644 .agents/skills/aiq-configure-workflow/SKILL.md create mode 100644 .agents/skills/aiq-configure-workflow/assets/config-scaffold.yml create mode 100644 .agents/skills/aiq-configure-workflow/references/composing-config.md create mode 100644 .agents/skills/aiq-configure-workflow/references/config-profiles.md create mode 100644 .agents/skills/aiq-configure-workflow/references/config-schema.md create mode 100644 .agents/skills/aiq-configure-workflow/references/env-vars.md create mode 100644 .agents/skills/aiq-configure-workflow/scripts/validate_config.py create mode 120000 .claude/skills/aiq-configure-workflow diff --git a/.agents/skills/README.md b/.agents/skills/README.md index 18c56beb7..c6b0fc09c 100644 --- a/.agents/skills/README.md +++ b/.agents/skills/README.md @@ -24,7 +24,7 @@ AI-Q has two distinct kinds of skill, separated by audience: | :-- | :-- | :-- | | **Audience** | Developers changing the AI-Q repo | Users calling a running AI-Q server | | **Location** | `.agents/skills/` (this directory) | top-level `skills/` | -| **Examples** | `aiq-add-data-source`, `aiq-add-tool`, `aiq-release-qa`, `aiq-prepare-pr`, `aiq-customize-prompts-models`, `aiq-maintain-ci` | `aiq-deploy`, `aiq-research` | +| **Examples** | `aiq-add-data-source`, `aiq-add-tool`, `aiq-configure-workflow`, `aiq-release-qa`, `aiq-prepare-pr`, `aiq-customize-prompts-models`, `aiq-maintain-ci` | `aiq-deploy`, `aiq-research` | | **Assumes** | A repo checkout and dev toolchain | A reachable AI-Q backend | Consumer skills under `skills/` are authored to be self-contained and exportable diff --git a/.agents/skills/aiq-add-data-source/SKILL.md b/.agents/skills/aiq-add-data-source/SKILL.md index ffc2f8ef8..a3adc487f 100644 --- a/.agents/skills/aiq-add-data-source/SKILL.md +++ b/.agents/skills/aiq-add-data-source/SKILL.md @@ -89,6 +89,7 @@ format failures for the new source package. ## Related Skills +- `aiq-configure-workflow` - `aiq-add-tool` - `aiq-release-qa` - `aiq-prepare-pr` diff --git a/.agents/skills/aiq-add-tool/SKILL.md b/.agents/skills/aiq-add-tool/SKILL.md index b573f40c7..db7955337 100644 --- a/.agents/skills/aiq-add-tool/SKILL.md +++ b/.agents/skills/aiq-add-tool/SKILL.md @@ -94,6 +94,7 @@ format failures for the new tool package. ## Related Skills +- `aiq-configure-workflow` - `aiq-add-data-source` - `aiq-release-qa` - `aiq-prepare-pr` diff --git a/.agents/skills/aiq-configure-workflow/SKILL.md b/.agents/skills/aiq-configure-workflow/SKILL.md new file mode 100644 index 000000000..1eaf986af --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/SKILL.md @@ -0,0 +1,89 @@ +--- +name: aiq-configure-workflow +description: Use when composing, adapting, or validating an AI-Q workflow YAML under configs/ — selecting a shipped profile, enabling tools and data_source_registry sources, wiring agents and the chat_deepresearcher_agent workflow, configuring general.telemetry (Phoenix, LangSmith, Weave, OTEL) and general.front_end aiq_api settings, and pre-flighting cross-references before deploy or serve. Hand off deploy to aiq-deploy, live research to aiq-research, prompt/model edits to aiq-customize-prompts-models, and new source code to aiq-add-tool or aiq-add-data-source. +license: Apache-2.0 +compatibility: Claude Code, Codex, Cursor, OpenCode, and Agent Skills-compatible tools. +metadata: + version: "0.1.0" + source-repo: "NVIDIA-AI-Blueprints/aiq" + tags: "aiq nemo-agent-toolkit config yaml workflow data-source-registry telemetry aiq_api" +allowed-tools: Read Bash Edit +--- + +# Configure AI-Q Workflows + +Use this skill when a developer or operator needs a **new `configs/config_*.yml`** +file. + +## Start Here + +- Confirm this is **config composition** — not deploy (`aiq-deploy`), live research + (`aiq-research`), prompt edits (`aiq-customize-prompts-models`), or new NAT + packages (`aiq-add-tool` / `aiq-add-data-source`). +- Copy the closest shipped `configs/*.yml` profile; merge feature blocks from others. +- **Every produced config must pass `validate_config.py` before hand-off.** + +## Authoritative References + +- `docs/source/customization/configuration-reference.md` — all fields and defaults +- `docs/source/customization/tools-and-sources.md` +- `docs/source/deployment/observability.md` — tracing setup detail +- `frontends/aiq_api/README.md` +- `configs/config_web_default_llamaindex.yml` / `configs/config_cli_default.yml` + +Bundle: + +- [references/config-profiles.md](references/config-profiles.md) — pick a starting profile. +- [references/composing-config.md](references/composing-config.md) — holistic config map + (`general`, `llms`, `functions`, `workflow`, telemetry, `aiq_api`) and how to tune them. +- [references/env-vars.md](references/env-vars.md) — environment variables by config feature. +- [references/config-schema.md](references/config-schema.md) — validator checks only. +- [assets/config-scaffold.yml](assets/config-scaffold.yml) — fallback scaffold. + +## Workflow + +1. **Scaffold** — `cp configs/.yml configs/config_.yml` (or + [assets/config-scaffold.yml](assets/config-scaffold.yml) + merge blocks). +2. **Compose** — [references/composing-config.md](references/composing-config.md): + adjust registry, tools, agents, LLMs, telemetry, `aiq_api`, workflow flags. + Use `config_web_default_llamaindex.yml` as the live default for web `general:` + blocks; `configuration-reference.md` for every option. Use + [references/env-vars.md](references/env-vars.md) for feature-specific env vars. +3. **Validate (required)** — + +```bash +uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py configs/config_.yml +``` + +Fix every `ERROR:`; re-run until exit code 0. Then hand off to `aiq-deploy` or: + +```bash +dotenv -f deploy/.env run nat serve --config_file configs/config_.yml --port 8000 +``` + +## Validation + +```bash +uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py +``` + +See [references/config-schema.md](references/config-schema.md). Expected: exit 0. + +## Common Mistakes + +- Skipping `validate_config.py` on a new config. +- Undefined `llms:` alias or registry tool not declared under `functions:`. +- Missing required workflow agents (`intent_classifier`, `shallow_research_agent`, + `deep_research_agent`). +- `use_async_deep_research: true` without `general.front_end` (`aiq_api`). +- Inventing feature YAML — copy from a shipped profile. + +## Related Skills + +- `aiq-deploy` +- `aiq-research` +- `aiq-customize-prompts-models` +- `aiq-add-tool` +- `aiq-add-data-source` +- `aiq-release-qa` +- `aiq-prepare-pr` diff --git a/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml b/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml new file mode 100644 index 000000000..8fc464dd3 --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Fallback scaffold only — prefer copying the closest configs/*.yml profile +# (see .agents/skills/aiq-configure-workflow/references/config-profiles.md). +# Merge feature blocks from other shipped profiles; do not invent fields. +# +# Usage: +# cp .agents/skills/aiq-configure-workflow/assets/config-scaffold.yml configs/my_workflow.yml + +general: + telemetry: + logging: + console: + _type: console + level: INFO + # Web/API: copy general.front_end from config_web_default_llamaindex.yml + +llms: + main_llm: + _type: nim + model_name: nvidia/nemotron-3-super-120b-a12b + base_url: "https://integrate.api.nvidia.com/v1" + temperature: 0.7 + max_tokens: 65536 + num_retries: 5 + +functions: + data_sources: + _type: data_source_registry + sources: [] # add sources — see aiq-add-data-source registry reference + + intent_classifier: + _type: intent_classifier + llm: main_llm + + shallow_research_agent: + _type: shallow_research_agent + llm: main_llm + + deep_research_agent: + _type: deep_research_agent + orchestrator_llm: main_llm + planner_llm: main_llm + researcher_llm: main_llm + writer_llm: main_llm + +workflow: + _type: chat_deepresearcher_agent + enable_escalation: true + enable_clarifier: true + checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db} diff --git a/.agents/skills/aiq-configure-workflow/references/composing-config.md b/.agents/skills/aiq-configure-workflow/references/composing-config.md new file mode 100644 index 000000000..60249ac95 --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/references/composing-config.md @@ -0,0 +1,207 @@ +# Composing a workflow config + +Holistic map for `configs/config_*.yml`. **Canonical field tables and defaults:** +`docs/source/customization/configuration-reference.md`. **Worked examples:** +`configs/config_web_default_llamaindex.yml` (web) and `configs/config_cli_default.yml` +(CLI). + +## How to edit + +1. Copy the nearest shipped profile ([config-profiles.md](config-profiles.md)). +2. Merge feature blocks from other `configs/*.yml` files (knowledge, guardrails, + sandbox, MCP auth) — do not invent fields. +3. Tune sections below; cross-check names against the canonical reference. +4. Run `validate_config.py` ([config-schema.md](config-schema.md)). + +Sibling skills for depth (do not duplicate here): + +- Registry / UI → `.agents/skills/aiq-add-data-source/references/registry-and-ui.md` +- LLM roles → `.agents/skills/aiq-customize-prompts-models/references/model-selection.md` +- New NAT packages → `aiq-add-tool` / `aiq-add-data-source` +- Secrets → `skills/aiq-deploy/references/env-and-secrets.md` + +## Top-level sections + +```yaml +general: # use_uvloop, telemetry, front_end (web only) +llms: # named aliases → referenced by agents +functions: # data_source_registry, tools, agents (+ optional feature functions) +workflow: # chat_deepresearcher_agent orchestrator flags +``` + +`${VAR}` / `${VAR:-default}` anywhere; secrets in `deploy/.env`. + +--- + +## `general:` + +| Area | CLI | Web (`nat serve`) | Where defaults live | +|------|-----|-------------------|---------------------| +| `use_uvloop` | optional | recommended `true` | `config_web_default_llamaindex.yml` | +| `telemetry` | yes | yes | both shipped profiles | +| `front_end` | **omit** | **required** (`aiq_api`) | `config_web_default_llamaindex.yml` | + +### Telemetry + +**Console** (always available): + +```yaml +general: + telemetry: + logging: + console: + _type: console + level: INFO # DEBUG | INFO | WARNING | ERROR +``` + +**Tracing** — enable under `general.telemetry.tracing` (exporters can coexist). +Uncomment the matching block in any `config_web_*.yml` or copy from +`docs/source/deployment/observability.md`: + +| Backend | YAML `_type` | Setup notes | +|---------|--------------|-------------| +| Phoenix | `phoenix` | `phoenix serve`; set `endpoint`, `project` | +| LangSmith | `langsmith` or env-only | `LANGCHAIN_TRACING_V2`, `LANGCHAIN_API_KEY`, `LANGCHAIN_PROJECT` | +| Weave | `weave` | `WANDB_API_KEY`; `project`, optional `redact_pii` | +| OpenTelemetry | `otelcollector_redaction` | `endpoint`; redaction + batch fields | + +`verbose: true` on `workflow:` or agents adds console detail without a tracer. + +### `front_end` (`aiq_api`) + +Required for UI, REST, WebSocket chat, async jobs, and `aiq-research`. Full +block with comments: `config_web_default_llamaindex.yml` lines 18–47. + +| Field | Default (web profile) | Tune when | +|-------|----------------------|-----------| +| `_type` | `aiq_api` | always | +| `runner_class` | `aiq_api.plugin.AIQAPIWorker` | rarely | +| `db_url` | `${NAT_JOB_STORE_DB_URL:-sqlite+aiosqlite:///./jobs.db}` | Postgres prod | +| `expiry_seconds` | `86400` (600–604800) | job retention policy | +| `cors` | localhost regex | production UI origin | + +**Env (not YAML):** `NAT_JOB_STORE_DB_URL`, `NAT_DASK_SCHEDULER_ADDRESS`, +`REQUIRE_AUTH`, `AIQ_TRACE_*` — `frontends/aiq_api/README.md`. + +`knowledge_retrieval` under `functions:` auto-enables `/v1/collections` and +`/v1/documents`. + +--- + +## `llms:` + +Define **aliases** under `llms:`; agents reference them by key (`llm`, +`orchestrator_llm`, `planner_llm`, etc.). Swap models by editing aliases or +repointing role fields — no Python changes. + +| Parameter | Typical use | Notes | +|-----------|-------------|-------| +| `_type` | `nim` or `openai` | Provider plugin | +| `model_name` | required | e.g. `nvidia/nemotron-3-super-120b-a12b` | +| `base_url` | NIM / compatible endpoint | Set explicitly for hosted NIM | +| `api_key` | optional | Falls back to `NVIDIA_API_KEY` for NIM | +| `temperature`, `top_p`, `max_tokens` | per role | See role table in configuration-reference | +| `num_retries` | resilience | default `5` | +| `chat_template_kwargs` | e.g. `enable_thinking: true` | chain-of-thought models | + +**Role-specific starting points** (temperature / max_tokens): intent classifier +(moderate), shallow researcher (low), deep orchestrator/writer (high + thinking), +summary LLM (short output). Full table: `configuration-reference.md` § +`llms` → "Common LLM Configurations". + +**Deeper guidance:** `docs/source/customization/swapping-models.md` and +`.agents/skills/aiq-customize-prompts-models/references/model-selection.md`. + +--- + +## `functions:` — tools, registry, agents + +Each entry is a named function under `functions:`. The YAML `_type` selects the +NAT plugin; the **key you choose** (e.g. `web_search_tool`) is what registry +`tools:` lists and agents reference. + +### Option index: retrieval / search tools + +Full parameter tables: `configuration-reference.md` § `functions`. Examples in +shipped configs under `configs/`. + +| `_type` | Purpose | Key options to tune | Env / profile | +|---------|---------|---------------------|---------------| +| `tavily_web_search` | Web search | `max_results`, `advanced_search`, `max_content_length`, `api_base_url` | `TAVILY_API_KEY`; all web profiles | +| `exa_web_search` | Web search (Exa) | `max_results`, `search_type` (`auto`/`fast`/`deep`), `full_text`, `highlights` | `EXA_API_KEY` | +| `paper_search` | Academic papers | `provider` (`serper`/`serpapi`/`searchapi`), `max_results` | `SERPER_API_KEY`, etc.; commented in most profiles | +| `knowledge_retrieval` | Document RAG | `backend` (`llamaindex`/`foundational_rag`/`opensearch`), `top_k`, `collection_name`, backend-specific URLs/auth | `config_web_default_llamaindex.yml`, `config_web_frag.yml`, `config_web_opensearch.yml` | + +Enable/disable for agents: register in `data_source_registry`, then inherit or +`exclude_tools` (below). Knowledge-layer backends: `docs/source/customization/knowledge-layer.md`. + +### `data_source_registry` + +Single source of truth for tools → UI toggles → agent inherit. One block with +`_type: data_source_registry` (often `data_sources`): + +```yaml + data_sources: + _type: data_source_registry + sources: + - id: web_search + name: "Web Search" + tools: [web_search_tool, advanced_web_search_tool] + web_search_tool: + _type: tavily_web_search + max_results: 5 +``` + +- Omit agent `tools:` → inherit all registry tools; use `exclude_tools` to specialize + (`config_web_default_llamaindex.yml` shallow vs deep). +- Enable a tool: declare under `functions:`, add to a source's `tools:`, set env vars. + +Optional **feature** function blocks (guardrails middleware, sandbox, skills, +domain catalog) are not search tools — copy YAML from the profile listed in +[config-profiles.md](config-profiles.md); field docs in `configuration-reference.md` +and feature guides under `docs/source/customization/`. + +### Option index: agents + +| `_type` | Key options to tune | Doc anchor | +|---------|---------------------|------------| +| `intent_classifier` | `llm`, `tools`, `llm_timeout`, `verbose` | `configuration-reference.md` § `intent_classifier` | +| `clarifier_agent` | `llm`, `max_turns`, `exclude_tools`, `verbose` | § `clarifier_agent` | +| `shallow_research_agent` | `llm`, `max_llm_turns`, `max_tool_iterations`, `exclude_tools` | § `shallow_research_agent` | +| `deep_research_agent` | role LLMs, `exclude_tools`, `enable_source_router`, `domain_catalog_path`, `enable_citation_verification`, `skills`, `sandbox`, concurrency caps | § `deep_research_agent` | + +### Agents (required by workflow) + +| Function | When | LLM fields | +|----------|------|------------| +| `intent_classifier` | always | `llm` | +| `shallow_research_agent` | always | `llm` | +| `deep_research_agent` | always | `orchestrator_llm`, `planner_llm`, `researcher_llm`, `writer_llm`, `source_router_llm` | +| `clarifier_agent` | `workflow.enable_clarifier: true` | `llm` | + +--- + +## `workflow:` + +```yaml +workflow: + _type: chat_deepresearcher_agent + enable_escalation: true # false → shallow only + enable_clarifier: true + use_async_deep_research: true # needs general.front_end + max_history: 20 + checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db} + verbose: true +``` + +Full defaults table: `configuration-reference.md` § `workflow`. + +--- + +## Validate + +```bash +uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py configs/config_.yml +``` + +What it checks: [config-schema.md](config-schema.md). Fix all errors before deploy. diff --git a/.agents/skills/aiq-configure-workflow/references/config-profiles.md b/.agents/skills/aiq-configure-workflow/references/config-profiles.md new file mode 100644 index 000000000..6835f319c --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/references/config-profiles.md @@ -0,0 +1,26 @@ +# Pick a starting profile + +There is **no single all-features profile**. Use the decision tree below, then copy +the file from `configs/`. For the full nine-profile table, see +`docs/source/customization/configuration-reference.md` ("Provided Config Files"). + +1. **Run mode** + - CLI only (`nat run`, `start_cli.sh`) → `config_cli_default.yml` (no `front_end`) + - Web UI / REST / async jobs / `aiq-research` → `config_web_*` or frontier/domain/skills profile (`front_end._type: aiq_api`) + +2. **Knowledge backend** + - None → `config_cli_default.yml` + - LlamaIndex → `config_web_default_llamaindex.yml` + - Foundational RAG → `config_web_frag.yml` (`RAG_SERVER_URL`, `RAG_INGEST_URL`) + - OpenSearch → `config_web_opensearch.yml` + +3. **Model family** + - Nemotron → most profiles + - GPT-5.2 orchestration/planning/writing → `config_frontier_models.yml` (`OPENAI_API_KEY`) + - GPT-OSS-120B → `config_web_default_guardrails.yml` or `config_openshell.yml` + +4. **Optional features** — copy blocks from: + - Guardrails → `config_web_default_guardrails.yml` + - MCP OAuth source → `config_web_frag_mcp_auth.yml` + - Domain routing + skills + Modal → `config_domain_routing_and_skills.yml` + - OpenShell sandbox → `config_openshell.yml` diff --git a/.agents/skills/aiq-configure-workflow/references/config-schema.md b/.agents/skills/aiq-configure-workflow/references/config-schema.md new file mode 100644 index 000000000..4ed02b8c0 --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/references/config-schema.md @@ -0,0 +1,46 @@ +# What `validate_config.py` checks + +Run on every new or edited `configs/config_*.yml` before deploy/serve. + +```bash +uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py +``` + +## Errors (exit 1) + +| Check | Meaning | +|-------|---------| +| LLM aliases | Every `llm`, `orchestrator_llm`, `planner_llm`, `researcher_llm`, `writer_llm`, `source_router_llm`, `summary_llm`, `intent_llm`, `summary_model` value must exist under `llms:` | +| Registry tools | Each tool in a `data_source_registry` source's `tools:` must be a key under `functions:` | +| Workflow type | `workflow._type` must be `chat_deepresearcher_agent` when `workflow:` is present | +| Required agents | `intent_classifier`, `shallow_research_agent`, `deep_research_agent` must exist under `functions:` | +| Clarifier | When `workflow.enable_clarifier` is true, `clarifier_agent` must exist under `functions:` | +| `front_end` type | When `general.front_end` is set, `_type` must be `aiq_api` | +| `aiq_api` settings | `expiry_seconds`, `db_url`, and `cors` shape are checked | +| Telemetry | Console logging and tracing exporter `_type` values are checked | +| Registry shape | `data_source_registry.sources` entries need `id`, `name`, and declared `tools` | + +## Warnings (exit 0) + +- No `llms:` block +- No `data_source_registry` +- `requires_auth: true` on a source (confirm MCP/OAuth wiring) +- `use_async_deep_research: true` without `general.front_end` +- LangSmith tracing without `LANGCHAIN_API_KEY` +- Weave tracing without `WANDB_API_KEY` + +## Env checklist + +Lists every `${VAR}` in the file and whether it is set in the current shell (not +values). Operator should align with `skills/aiq-deploy/references/env-and-secrets.md`. + +## Top-level shape + +```yaml +general: # telemetry; front_end (web) +llms: +functions: # registry, tools, agents +workflow: # chat_deepresearcher_agent +``` + +Field reference: `docs/source/customization/configuration-reference.md`. diff --git a/.agents/skills/aiq-configure-workflow/references/env-vars.md b/.agents/skills/aiq-configure-workflow/references/env-vars.md new file mode 100644 index 000000000..77e06f02d --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/references/env-vars.md @@ -0,0 +1,57 @@ +# Environment variables for config authors + +Use this as a quick map while composing `configs/config_*.yml`. Keep secret +values in `deploy/.env`; never paste values into config files or PR text. + +Canonical references: + +- `deploy/.env.example` +- `frontends/aiq_api/README.md` +- `docs/source/deployment/observability.md` +- profile comments in `configs/config_*.yml` + +## Core runtime + +| Variable | When needed | Notes | +|----------|-------------|-------| +| `NVIDIA_API_KEY` | Default NIM-backed model profiles | Required for model calls unless the config uses a different model provider. | +| `OPENAI_API_KEY` | Frontier/OpenAI model profiles | Required by `config_frontier_models.yml`. | +| `NAT_JOB_STORE_DB_URL` | Web/API mode | Job, event, and artifact metadata database. Defaults to local SQLite. Use PostgreSQL for production. | +| `NAT_DASK_SCHEDULER_ADDRESS` | Distributed execution | Optional. A local Dask cluster is created when unset. | + +## Retrieval and tools + +| Variable | Enables | +|----------|---------| +| `TAVILY_API_KEY` | Tavily web search | +| `EXA_API_KEY` | Exa web search | +| `SERPER_API_KEY` | Serper paper search | +| `SERPAPI_API_KEY` | SerpAPI paper search | +| `SEARCHAPI_API_KEY` | SearchAPI paper search | +| `RAG_SERVER_URL`, `RAG_INGEST_URL` | Foundational RAG profiles | + +## Web API, auth, and tracing + +| Variable | When needed | +|----------|-------------| +| `REQUIRE_AUTH` | Enforce API authentication. Requires validator registration. | +| `AIQ_TRACE_USER_IDENTITY_MODE`, `AIQ_TRACE_USER_IDENTITY_HMAC_SECRET` | User identity tagging for NAT spans. | +| `AIQ_TRACE_CLIENT_ID_MODE`, `AIQ_TRACE_CLIENT_ID_HMAC_SECRET`, `AIQ_TRACE_CLIENT_IP_HEADERS` | Client tagging for NAT spans. | +| `LANGCHAIN_TRACING_V2`, `LANGCHAIN_API_KEY`, `LANGCHAIN_PROJECT` | LangSmith tracing. | +| `WANDB_API_KEY` | Weave tracing. | + +## Sandbox and artifact storage + +| Variable | When needed | +|----------|-------------| +| `AIQ_OPENSHELL_GATEWAY_NAME`, `AIQ_OPENSHELL_IMAGE`, `AIQ_OPENSHELL_POLICY_FILE` | OpenShell sandbox profile. | +| `AIQ_ARTIFACT_BLOB_PROVIDER`, `AIQ_ARTIFACT_S3_BUCKET`, `AIQ_ARTIFACT_S3_ENDPOINT_URL`, `AIQ_ARTIFACT_S3_REGION`, `AIQ_ARTIFACT_S3_PREFIX` | Optional object storage for sandbox artifacts. | + +## Validate without leaking + +```bash +uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py configs/config_.yml +``` + +The validator prints variable names and whether they are set in the current +shell; it does not print secret values. diff --git a/.agents/skills/aiq-configure-workflow/scripts/validate_config.py b/.agents/skills/aiq-configure-workflow/scripts/validate_config.py new file mode 100644 index 000000000..12f59596a --- /dev/null +++ b/.agents/skills/aiq-configure-workflow/scripts/validate_config.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Static, read-only validator for an AI-Q workflow YAML config. + +Checks cross-references and workflow shape before a config reaches a running +backend. Dependency-light: PyYAML + stdlib only. + +Usage: + uv run python validate_config.py path/to/config.yml +""" + +from __future__ import annotations + +import os +import re +import sys + +try: + import yaml +except ImportError: # pragma: no cover + print("ERROR: PyYAML is required. Install with: uv sync", file=sys.stderr) + sys.exit(2) + +WORKFLOW_TYPE = "chat_deepresearcher_agent" +FRONT_END_TYPE = "aiq_api" +LOG_LEVELS = {"DEBUG", "INFO", "WARNING", "ERROR"} +TRACING_TYPES = {"langsmith", "otelcollector_redaction", "phoenix", "weave"} +EXPIRY_SECONDS_MIN = 600 +EXPIRY_SECONDS_MAX = 604800 + +LLM_REF_FIELDS = ( + "llm", + "orchestrator_llm", + "planner_llm", + "researcher_llm", + "writer_llm", + "source_router_llm", + "summary_llm", + "intent_llm", + "summary_model", +) + +REQUIRED_WORKFLOW_AGENTS = ( + "intent_classifier", + "shallow_research_agent", + "deep_research_agent", +) + +ENV_REF = re.compile(r"\$\{([A-Z0-9_]+)(?::-[^}]*)?\}") + + +def _load(path: str) -> tuple[dict, str]: + with open(path, encoding="utf-8") as fh: + text = fh.read() + data = yaml.safe_load(text) + if not isinstance(data, dict): + msg = f"Top-level YAML must be a mapping (got {type(data).__name__})." + raise ValueError(msg) + return data, text + + +def _iter_refs(node: object): + """Yield (field, alias) for every LLM-alias reference found recursively.""" + if isinstance(node, dict): + for key, value in node.items(): + if key in LLM_REF_FIELDS and isinstance(value, str): + yield key, value + else: + yield from _iter_refs(value) + elif isinstance(node, list): + for item in node: + yield from _iter_refs(item) + + +def _general_block(data: dict) -> dict: + general = data.get("general") or {} + return general if isinstance(general, dict) else {} + + +def _validate_registry(registry: dict, declared_functions: set[str], errors: list[str], warnings: list[str]) -> None: + sources = registry.get("sources") + if not isinstance(sources, list): + errors.append("data_source_registry.sources must be a list.") + return + + for index, source in enumerate(sources): + if not isinstance(source, dict): + errors.append(f"data_source_registry.sources[{index}] must be a mapping.") + continue + sid = source.get("id") + label = sid or f"" + if not sid: + errors.append(f"data_source_registry source at index {index} is missing id.") + if not source.get("name"): + errors.append(f"source '{label}' is missing name.") + tools = source.get("tools") + if tools is None: + errors.append(f"source '{label}' is missing tools:.") + tools = [] + elif not isinstance(tools, list): + errors.append(f"source '{label}' tools: must be a list.") + tools = [] + for tool in tools: + if tool not in declared_functions: + errors.append( + f"source '{label}' lists tool '{tool}' in its tools:, but '{tool}' is not declared under functions:" + ) + for bool_field in ("default_enabled", "requires_auth"): + if bool_field in source and not isinstance(source[bool_field], bool): + errors.append(f"source '{label}' {bool_field}: must be true or false.") + if source.get("requires_auth") is True: + warnings.append( + f"source '{label}' has requires_auth: true — confirm auth/token wiring " + "(see config_web_frag_mcp_auth.yml)." + ) + + +def _validate_telemetry(general: dict, errors: list[str], warnings: list[str]) -> None: + telemetry = general.get("telemetry") + if telemetry is None: + return + if not isinstance(telemetry, dict): + errors.append("general.telemetry must be a mapping.") + return + + logging = telemetry.get("logging") + if logging is not None: + if not isinstance(logging, dict): + errors.append("general.telemetry.logging must be a mapping.") + else: + console = logging.get("console") + if console is not None: + if not isinstance(console, dict): + errors.append("general.telemetry.logging.console must be a mapping.") + else: + if console.get("_type") != "console": + errors.append("general.telemetry.logging.console._type must be 'console'.") + level = console.get("level") + if level is not None and str(level).upper() not in LOG_LEVELS: + errors.append( + f"general.telemetry.logging.console.level must be one of {', '.join(sorted(LOG_LEVELS))}." + ) + + tracing = telemetry.get("tracing") + if tracing is not None: + if not isinstance(tracing, dict): + errors.append("general.telemetry.tracing must be a mapping.") + return + for name, exporter in tracing.items(): + if not isinstance(exporter, dict): + errors.append(f"general.telemetry.tracing.{name} must be a mapping.") + continue + exporter_type = exporter.get("_type") + if exporter_type not in TRACING_TYPES: + errors.append( + f"general.telemetry.tracing.{name}._type must be one of {', '.join(sorted(TRACING_TYPES))}." + ) + if exporter_type in {"otelcollector_redaction", "phoenix"} and not exporter.get("endpoint"): + warnings.append(f"tracing exporter '{name}' usually needs an endpoint.") + if exporter_type == "langsmith" and not os.environ.get("LANGCHAIN_API_KEY"): + warnings.append("LangSmith tracing is configured; confirm LANGCHAIN_API_KEY is set.") + if exporter_type == "weave" and not os.environ.get("WANDB_API_KEY"): + warnings.append("Weave tracing is configured; confirm WANDB_API_KEY is set.") + + +def _validate_front_end(general: dict, errors: list[str]) -> None: + front_end = general.get("front_end") + if front_end is None: + return + if not isinstance(front_end, dict): + errors.append("`general.front_end` must be a mapping.") + return + if front_end.get("_type") != FRONT_END_TYPE: + errors.append(f"general.front_end._type must be '{FRONT_END_TYPE}' for the AI-Q web API.") + db_url = front_end.get("db_url") + if db_url is not None and not isinstance(db_url, str): + errors.append("general.front_end.db_url must be a string.") + expiry = front_end.get("expiry_seconds") + if expiry is not None: + if not isinstance(expiry, int): + errors.append("general.front_end.expiry_seconds must be an integer.") + elif expiry < EXPIRY_SECONDS_MIN or expiry > EXPIRY_SECONDS_MAX: + errors.append( + f"general.front_end.expiry_seconds must be between {EXPIRY_SECONDS_MIN} and {EXPIRY_SECONDS_MAX}." + ) + cors = front_end.get("cors") + if cors is not None and not isinstance(cors, dict): + errors.append("general.front_end.cors must be a mapping.") + + +def validate(path: str) -> int: + errors: list[str] = [] + warnings: list[str] = [] + + try: + data, raw = _load(path) + except FileNotFoundError: + print(f"ERROR: file not found: {path}", file=sys.stderr) + return 2 + except (yaml.YAMLError, ValueError) as exc: + print(f"ERROR: could not parse YAML: {exc}", file=sys.stderr) + return 2 + + llms = data.get("llms") or {} + defined_aliases = set(llms.keys()) if isinstance(llms, dict) else set() + + functions = data.get("functions") or {} + if not isinstance(functions, dict): + functions = {} + declared_functions = set(functions.keys()) + + for field, alias in _iter_refs(functions): + if alias not in defined_aliases: + defined = ", ".join(sorted(defined_aliases)) or "none" + errors.append( + f"llm alias '{alias}' referenced by a '{field}' field is not defined under llms: (defined: {defined})" + ) + + if not defined_aliases: + warnings.append("no `llms:` block found — the config defines no LLM aliases.") + + registry = None + for block in functions.values(): + if isinstance(block, dict) and block.get("_type") == "data_source_registry": + registry = block + break + + if registry is None: + warnings.append("no data_source_registry function found (fine for minimal configs).") + else: + _validate_registry(registry, declared_functions, errors, warnings) + + workflow = data.get("workflow") + if workflow is not None: + if not isinstance(workflow, dict): + errors.append("`workflow:` must be a mapping.") + else: + wf_type = workflow.get("_type") + if wf_type != WORKFLOW_TYPE: + errors.append(f"workflow._type must be '{WORKFLOW_TYPE}' (got {wf_type!r}).") + for agent_name in REQUIRED_WORKFLOW_AGENTS: + if agent_name not in declared_functions: + errors.append(f"workflow requires function '{agent_name}' under functions: (missing).") + if workflow.get("enable_clarifier") is True and "clarifier_agent" not in declared_functions: + errors.append("workflow.enable_clarifier is true but 'clarifier_agent' is missing under functions:.") + if workflow.get("use_async_deep_research") is True and "front_end" not in _general_block(data): + warnings.append( + "use_async_deep_research is true but general.front_end is missing " + "(web/aiq_api mode expected for async jobs)." + ) + + general = _general_block(data) + _validate_telemetry(general, errors, warnings) + _validate_front_end(general, errors) + + env_vars = sorted(set(ENV_REF.findall(raw))) + + print(f"AI-Q config validation: {path}") + print("-" * 60) + for err in errors: + print(f"ERROR: {err}") + for warn in warnings: + print(f"WARN: {warn}") + + if env_vars: + print("\nEnvironment variables referenced (set these in deploy/.env):") + for var in env_vars: + present = "set" if os.environ.get(var) else "NOT set in this shell" + print(f" - {var} ({present})") + + print("-" * 60) + if errors: + print(f"RESULT: {len(errors)} error(s), {len(warnings)} warning(s). Fix errors before deploying.") + return 1 + print(f"RESULT: no errors, {len(warnings)} warning(s). Config is structurally valid.") + return 0 + + +def main(argv: list[str] | None = None) -> int: + args = argv if argv is not None else sys.argv + if len(args) != 2: + print("Usage: validate_config.py path/to/config.yml", file=sys.stderr) + return 2 + return validate(args[1]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/aiq-customize-prompts-models/SKILL.md b/.agents/skills/aiq-customize-prompts-models/SKILL.md index fb909e079..9a59daf8a 100644 --- a/.agents/skills/aiq-customize-prompts-models/SKILL.md +++ b/.agents/skills/aiq-customize-prompts-models/SKILL.md @@ -112,6 +112,7 @@ smoke run is the real check; a config/prompt-only change needs no Python lint. ## Related Skills +- `aiq-configure-workflow` - `aiq-add-tool` - `aiq-add-data-source` - `aiq-release-qa` diff --git a/.claude/skills/aiq-configure-workflow b/.claude/skills/aiq-configure-workflow new file mode 120000 index 000000000..8fadc8b14 --- /dev/null +++ b/.claude/skills/aiq-configure-workflow @@ -0,0 +1 @@ +../../.agents/skills/aiq-configure-workflow \ No newline at end of file From 843f94793961b1fc8aea835201305cae2006cce7 Mon Sep 17 00:00:00 2001 From: Soumili Nandi Date: Tue, 14 Jul 2026 12:26:12 -0700 Subject: [PATCH 2/2] fix(aiq-configure-workflow): address validator feedback Signed-off-by: Soumili Nandi --- .../assets/config-scaffold.yml | 2 +- .../references/config-schema.md | 2 +- .../references/env-vars.md | 1 + .../scripts/validate_config.py | 42 +++++++++++-------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml b/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml index 8fc464dd3..504fc9255 100644 --- a/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml +++ b/.agents/skills/aiq-configure-workflow/assets/config-scaffold.yml @@ -48,5 +48,5 @@ functions: workflow: _type: chat_deepresearcher_agent enable_escalation: true - enable_clarifier: true + enable_clarifier: false # set true and add clarifier_agent under functions: to enable checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db} diff --git a/.agents/skills/aiq-configure-workflow/references/config-schema.md b/.agents/skills/aiq-configure-workflow/references/config-schema.md index 4ed02b8c0..76fe5231a 100644 --- a/.agents/skills/aiq-configure-workflow/references/config-schema.md +++ b/.agents/skills/aiq-configure-workflow/references/config-schema.md @@ -12,7 +12,7 @@ uv run python .agents/skills/aiq-configure-workflow/scripts/validate_config.py < |-------|---------| | LLM aliases | Every `llm`, `orchestrator_llm`, `planner_llm`, `researcher_llm`, `writer_llm`, `source_router_llm`, `summary_llm`, `intent_llm`, `summary_model` value must exist under `llms:` | | Registry tools | Each tool in a `data_source_registry` source's `tools:` must be a key under `functions:` | -| Workflow type | `workflow._type` must be `chat_deepresearcher_agent` when `workflow:` is present | +| Workflow | `workflow:` must exist and `_type` must be `chat_deepresearcher_agent` | | Required agents | `intent_classifier`, `shallow_research_agent`, `deep_research_agent` must exist under `functions:` | | Clarifier | When `workflow.enable_clarifier` is true, `clarifier_agent` must exist under `functions:` | | `front_end` type | When `general.front_end` is set, `_type` must be `aiq_api` | diff --git a/.agents/skills/aiq-configure-workflow/references/env-vars.md b/.agents/skills/aiq-configure-workflow/references/env-vars.md index 77e06f02d..48b25e123 100644 --- a/.agents/skills/aiq-configure-workflow/references/env-vars.md +++ b/.agents/skills/aiq-configure-workflow/references/env-vars.md @@ -18,6 +18,7 @@ Canonical references: | `OPENAI_API_KEY` | Frontier/OpenAI model profiles | Required by `config_frontier_models.yml`. | | `NAT_JOB_STORE_DB_URL` | Web/API mode | Job, event, and artifact metadata database. Defaults to local SQLite. Use PostgreSQL for production. | | `NAT_DASK_SCHEDULER_ADDRESS` | Distributed execution | Optional. A local Dask cluster is created when unset. | +| `AIQ_CHECKPOINT_DB` | `workflow.checkpoint_db` | Optional. Defaults to local SQLite file `./checkpoints.db`. | ## Retrieval and tools diff --git a/.agents/skills/aiq-configure-workflow/scripts/validate_config.py b/.agents/skills/aiq-configure-workflow/scripts/validate_config.py index 12f59596a..f26013931 100644 --- a/.agents/skills/aiq-configure-workflow/scripts/validate_config.py +++ b/.agents/skills/aiq-configure-workflow/scripts/validate_config.py @@ -63,6 +63,7 @@ def _load(path: str) -> tuple[dict, str]: + """Load a YAML file and return its parsed mapping plus raw text.""" with open(path, encoding="utf-8") as fh: text = fh.read() data = yaml.safe_load(text) @@ -86,11 +87,13 @@ def _iter_refs(node: object): def _general_block(data: dict) -> dict: + """Return the top-level general block when it is a mapping.""" general = data.get("general") or {} return general if isinstance(general, dict) else {} def _validate_registry(registry: dict, declared_functions: set[str], errors: list[str], warnings: list[str]) -> None: + """Validate data source registry shape and tool references.""" sources = registry.get("sources") if not isinstance(sources, list): errors.append("data_source_registry.sources must be a list.") @@ -129,6 +132,7 @@ def _validate_registry(registry: dict, declared_functions: set[str], errors: lis def _validate_telemetry(general: dict, errors: list[str], warnings: list[str]) -> None: + """Validate telemetry logging and tracing configuration shape.""" telemetry = general.get("telemetry") if telemetry is None: return @@ -177,6 +181,7 @@ def _validate_telemetry(general: dict, errors: list[str], warnings: list[str]) - def _validate_front_end(general: dict, errors: list[str]) -> None: + """Validate AI-Q API front-end settings.""" front_end = general.get("front_end") if front_end is None: return @@ -202,6 +207,7 @@ def _validate_front_end(general: dict, errors: list[str]) -> None: def validate(path: str) -> int: + """Validate one AI-Q workflow config and print a human-readable report.""" errors: list[str] = [] warnings: list[str] = [] @@ -244,23 +250,24 @@ def validate(path: str) -> int: _validate_registry(registry, declared_functions, errors, warnings) workflow = data.get("workflow") - if workflow is not None: - if not isinstance(workflow, dict): - errors.append("`workflow:` must be a mapping.") - else: - wf_type = workflow.get("_type") - if wf_type != WORKFLOW_TYPE: - errors.append(f"workflow._type must be '{WORKFLOW_TYPE}' (got {wf_type!r}).") - for agent_name in REQUIRED_WORKFLOW_AGENTS: - if agent_name not in declared_functions: - errors.append(f"workflow requires function '{agent_name}' under functions: (missing).") - if workflow.get("enable_clarifier") is True and "clarifier_agent" not in declared_functions: - errors.append("workflow.enable_clarifier is true but 'clarifier_agent' is missing under functions:.") - if workflow.get("use_async_deep_research") is True and "front_end" not in _general_block(data): - warnings.append( - "use_async_deep_research is true but general.front_end is missing " - "(web/aiq_api mode expected for async jobs)." - ) + for agent_name in REQUIRED_WORKFLOW_AGENTS: + if agent_name not in declared_functions: + errors.append(f"workflow requires function '{agent_name}' under functions: (missing).") + if workflow is None: + errors.append("`workflow:` is required for an AI-Q workflow config.") + elif not isinstance(workflow, dict): + errors.append("`workflow:` must be a mapping.") + else: + wf_type = workflow.get("_type") + if wf_type != WORKFLOW_TYPE: + errors.append(f"workflow._type must be '{WORKFLOW_TYPE}' (got {wf_type!r}).") + if workflow.get("enable_clarifier") is True and "clarifier_agent" not in declared_functions: + errors.append("workflow.enable_clarifier is true but 'clarifier_agent' is missing under functions:.") + if workflow.get("use_async_deep_research") is True and not _general_block(data).get("front_end"): + warnings.append( + "use_async_deep_research is true but general.front_end is missing " + "(web/aiq_api mode expected for async jobs)." + ) general = _general_block(data) _validate_telemetry(general, errors, warnings) @@ -290,6 +297,7 @@ def validate(path: str) -> int: def main(argv: list[str] | None = None) -> int: + """Parse command-line arguments and run validation.""" args = argv if argv is not None else sys.argv if len(args) != 2: print("Usage: validate_config.py path/to/config.yml", file=sys.stderr)