fix: restore 2.2.1 release readiness

.dockerignore

@@ -62,6 +62,7 @@ tests/
 !mcp/README.md
 !sources/knowledge_layer/README.md
 !sources/tavily_web_search/README.md
+!src/aiq_agent/agents/deep_researcher/skills/**/SKILL.md
 docs/
 license_report.pdf
 

.github/workflows/ci.yml

@@ -227,6 +227,11 @@ jobs:
           chmod +x ci/scripts/test_scripts.sh
           ci/scripts/test_scripts.sh --skip-setup
 
+      - name: Validate VCS-less AI-Q release artifact on Python 3.11
+        env:
+          UV_PYTHON: "3.11"
+        run: ci/scripts/test_release_artifact.sh
+
       - name: Create MCP production environment
         run: |
           set -euo pipefail

ci/scripts/test_release_artifact.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Validate the installable AI-Q artifact without relying on checkout metadata or
+# editable imports. CI intentionally runs this against the committed HEAD.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+TMP_ROOT="$(mktemp -d)"
+SOURCE_ROOT="$TMP_ROOT/source"
+WHEEL_DIR="$TMP_ROOT/wheel"
+VENV_DIR="$TMP_ROOT/venv"
+RUNTIME_ROOT="$TMP_ROOT/runtime"
+
+cleanup() {
+    rm -rf -- "$TMP_ROOT"
+}
+trap cleanup EXIT
+
+mkdir -p "$SOURCE_ROOT" "$WHEEL_DIR" "$RUNTIME_ROOT"
+git -C "$REPO_ROOT" archive HEAD | tar -xf - -C "$SOURCE_ROOT"
+if [[ -e "$SOURCE_ROOT/.git" ]]; then
+    echo "Release source export unexpectedly contains .git metadata" >&2
+    exit 1
+fi
+
+uv build --wheel --out-dir "$WHEEL_DIR" "$SOURCE_ROOT"
+wheel_count=$(find "$WHEEL_DIR" -maxdepth 1 -type f -name 'aiq_agent-*.whl' | wc -l | tr -d ' ')
+if [[ "$wheel_count" != "1" ]]; then
+    echo "Expected exactly one aiq-agent wheel, found $wheel_count" >&2
+    exit 1
+fi
+WHEEL_PATH=$(find "$WHEEL_DIR" -maxdepth 1 -type f -name 'aiq_agent-*.whl')
+
+UV_PROJECT_ENVIRONMENT="$VENV_DIR" \
+    uv sync --project "$SOURCE_ROOT" --frozen --group dev --no-editable --no-install-package aiq-agent
+uv pip install --python "$VENV_DIR/bin/python" --no-deps --reinstall "$WHEEL_PATH"
+uv pip check --python "$VENV_DIR/bin/python"
+
+cp -R "$SOURCE_ROOT/configs" "$RUNTIME_ROOT/configs"
+cd "$RUNTIME_ROOT"
+
+"$VENV_DIR/bin/python" -I - "$SOURCE_ROOT" "$WHEEL_PATH" <<'PY'
+import importlib.metadata
+import sys
+import zipfile
+from pathlib import Path
+
+import aiq_agent
+
+source_root = Path(sys.argv[1]).resolve()
+wheel_path = Path(sys.argv[2]).resolve()
+expected = {
+    "aiq_agent/agents/chat_researcher/prompts/context_aware_intent_router.j2",
+    "aiq_agent/agents/chat_researcher/prompts/intent_classification.j2",
+    "aiq_agent/agents/clarifier/prompts/research_clarification.j2",
+    "aiq_agent/agents/deep_researcher/prompts/orchestrator.j2",
+    "aiq_agent/agents/deep_researcher/prompts/planner.j2",
+    "aiq_agent/agents/deep_researcher/prompts/researcher.j2",
+    "aiq_agent/agents/deep_researcher/prompts/source_registry.j2",
+    "aiq_agent/agents/deep_researcher/prompts/source_router.j2",
+    "aiq_agent/agents/deep_researcher/prompts/writer.j2",
+    "aiq_agent/agents/report_rewriter/prompts/edit.j2",
+    "aiq_agent/agents/shallow_researcher/prompts/researcher.j2",
+    "aiq_agent/agents/deep_researcher/skills/research/data-table-analysis/SKILL.md",
+    "aiq_agent/agents/deep_researcher/skills/research/forecast-analysis/SKILL.md",
+    "aiq_agent/agents/deep_researcher/skills/research/lightweight-calculation/SKILL.md",
+    "aiq_agent/agents/deep_researcher/skills/synthesis/long-form-report-writer/SKILL.md",
+    "aiq_agent/agents/deep_researcher/skills/synthesis/prediction-report-writer/SKILL.md",
+    "aiq_agent/agents/deep_researcher/skills/visualization/chart-generation/SKILL.md",
+}
+
+source_package = source_root / "src" / "aiq_agent"
+actual_source = {
+    path.relative_to(source_root / "src").as_posix()
+    for path in (source_package / "agents").glob("*/prompts/*.j2")
+}
+actual_source.update(
+    path.relative_to(source_root / "src").as_posix()
+    for path in (source_package / "agents" / "deep_researcher" / "skills").rglob("SKILL.md")
+)
+if actual_source != expected:
+    raise SystemExit(
+        "Runtime asset manifest is stale: "
+        f"missing={sorted(expected - actual_source)}, unexpected={sorted(actual_source - expected)}"
+    )
+
+for relative in expected:
+    source_path = source_root / "src" / relative
+    if source_path.stat().st_size == 0:
+        raise SystemExit(f"Source runtime asset is empty: {relative}")
+
+with zipfile.ZipFile(wheel_path) as wheel:
+    wheel_entries = {entry.filename: entry.file_size for entry in wheel.infolist()}
+for relative in expected:
+    if wheel_entries.get(relative, 0) == 0:
+        raise SystemExit(f"Wheel runtime asset is missing or empty: {relative}")
+
+package_root = Path(aiq_agent.__file__).resolve().parent
+venv_root = Path(sys.prefix).resolve()
+try:
+    package_root.relative_to(venv_root)
+except ValueError as exc:
+    raise SystemExit(f"aiq_agent imported outside the isolated environment: {package_root}") from exc
+if source_root in package_root.parents:
+    raise SystemExit(f"aiq_agent imported from the source export: {package_root}")
+
+for relative in expected:
+    installed_path = package_root / Path(relative).relative_to("aiq_agent")
+    if not installed_path.is_file() or installed_path.stat().st_size == 0:
+        raise SystemExit(f"Installed runtime asset is missing or empty: {relative}")
+
+direct_url = importlib.metadata.distribution("aiq-agent").read_text("direct_url.json") or ""
+if '"editable": true' in direct_url:
+    raise SystemExit("aiq-agent was installed editable")
+
+print(f"Verified {len(expected)} non-empty runtime assets in source, wheel, and installed distribution")
+print(f"Verified isolated aiq_agent import: {aiq_agent.__file__}")
+PY
+
+export NVIDIA_API_KEY="ci-not-a-real-key"  # pragma: allowlist secret
+export OPENAI_API_KEY="ci-not-a-real-key"  # pragma: allowlist secret
+export TAVILY_API_KEY="ci-not-a-real-key"  # pragma: allowlist secret
+export SERPER_API_KEY="ci-not-a-real-key"  # pragma: allowlist secret
+export REDIS_PASSWORD="ci-not-a-real-password"  # pragma: allowlist secret
+export NAT_JOB_STORE_DB_URL="sqlite+aiosqlite:///$RUNTIME_ROOT/jobs.db"
+export AIQ_CHECKPOINT_DB="$RUNTIME_ROOT/checkpoints.db"
+export AIQ_SUMMARY_DB="sqlite+aiosqlite:///$RUNTIME_ROOT/summaries.db"
+export AIQ_CHROMA_DIR="$RUNTIME_ROOT/chroma"
+export AZURE_SEARCH_ENDPOINT="https://azure-search.invalid"
+export MCP_TOKEN_DB="$RUNTIME_ROOT/mcp_tokens.db"
+export AIQ_OPENSHELL_POLICY_FILE="$RUNTIME_ROOT/configs/openshell/aiq-research-policy.yaml"
+
+config_count=0
+while IFS= read -r config_path; do
+    config_count=$((config_count + 1))
+    echo "Validating installed workflow: ${config_path#"$RUNTIME_ROOT/"}"
+    "$VENV_DIR/bin/nat" validate --config_file "$config_path"
+done < <(find "$RUNTIME_ROOT/configs" -maxdepth 1 -type f -name 'config_*.yml' | sort)
+
+if [[ "$config_count" -eq 0 ]]; then
+    echo "No shipped top-level workflow configs were discovered" >&2
+    exit 1
+fi
+echo "Validated $config_count shipped top-level workflow configs from the installed release artifact"

docs/source/customization/knowledge-layer.md

@@ -281,8 +281,6 @@ Persisted vector stores are tied to both the embedding model and its output dime
 - **Azure AI Search:** model and dimension are part of the physical index identity; changing either creates an isolated
   index that must be populated by re-ingestion.
 
-See the detailed [knowledge-layer setup migration procedure](../../../sources/knowledge_layer/KNOWLEDGE-LAYER-SETUP.md#migrating-an-embedding-model).
-
 OpenSearch ingestion is text-only: it extracts text from PDF, DOCX, PPTX, and supported plain-text formats, but does not
 perform LlamaIndex table/image/chart extraction. Distributed Dask ingestion also disables document-summary generation
 because the configured summary LLM is not serialized to workers; use local ingestion when summaries are required.

docs/source/deployment/docker-compose.md

@@ -164,7 +164,7 @@ See [Production Considerations](./production.md#s3-security-responsibility).
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `backend_url` | `http://aiq-agent:8000` | Backend API URL as seen from the frontend container. |
+| `BACKEND_URL` | `http://aiq-agent:8000` | Backend API URL as seen from the frontend container. |
 
 ### Dask Worker Settings
 

docs/source/deployment/observability.md

@@ -109,13 +109,25 @@ instrument LangChain directly can present a different tree.
 
 ## Weights & Biases Weave
 
-[Weave](https://wandb.ai/site/weave) provides experiment tracking and trace logging integrated with the Weights & Biases platform. NAT includes Weave support via the `weave` extra (`nvidia-nat[weave]`), which is already installed in this project.
+[Weave](https://wandb.ai/site/weave) provides experiment tracking and trace
+logging integrated with the Weights & Biases platform. Weave support is an
+optional NAT extra and is not installed by default.
 
 ### Setup
 
-1. Create a [Weights & Biases](https://wandb.ai/) account if you do not have one.
+1. Install the exporter into your local environment:
 
-2. Set the API key in `deploy/.env`:
+   ```bash
+   uv pip install "nvidia-nat[weave]==1.8.0"
+   ```
+
+   For production or container deployments, add this exact pinned dependency
+   to the image build and rebuild the image. Installing it into a running
+   container is not a durable deployment.
+
+2. Create a [Weights & Biases](https://wandb.ai/) account if you do not have one.
+
+3. Set the API key in `deploy/.env`:
 
    ```bash
    WANDB_API_KEY=your-wandb-api-key
@@ -127,7 +139,7 @@ instrument LangChain directly can present a different tree.
    wandb login
    ```
 
-3. Enable Weave tracing in your YAML config:
+4. Enable Weave tracing in your YAML config:
 
    ```yaml
    general:

docs/source/deployment/production.md

@@ -199,15 +199,15 @@ production artifact storage.
 
 ### Horizontal Backend Scaling
 
-The backend is stateless apart from database connections, so it can be horizontally scaled behind a load balancer.
+The shipped Docker Compose topology supports one backend instance. Do not use
+Compose service scaling for production because the stack does not provide the
+required backend load balancer or shared scheduler topology.
 
-**Docker Compose:** Run multiple backend containers by scaling the service and using a reverse proxy (such as Traefik or NGINX) in front:
+For production horizontal scaling, deploy with Helm and set
+`aiq.apps.backend.replicas` or the `aiq.apps.backend.autoscaling` values. Refer
+to [Kubernetes and Helm](./kubernetes.md) for the supported deployment path.
 
-```bash
-docker compose --env-file ../.env -f docker-compose.yaml up -d --scale aiq-agent=3
-```
-
-Note that each scaled instance starts its own embedded Dask scheduler and worker.
+Each backend replica starts its own embedded Dask scheduler and worker.
 The shipped container entrypoint always creates that embedded cluster. A deployment
 that uses a shared Dask cluster must provide a custom entrypoint (for example,
 starting `/app/deploy/start_web.py` directly), set

docs/source/evaluation/benchmarks/deep-research-bench.md

@@ -19,6 +19,7 @@ SPDX-License-Identifier: Apache-2.0
 
 ```bash
 export TAVILY_API_KEY=your_key              # For web search
+export SERPER_API_KEY=your_key              # For Google Scholar paper search
 export NVIDIA_API_KEY=your_key              # For agent execution (integrate.api.nvidia.com)
 export OPENAI_API_KEY=your_key              # For frontier model in config (optional)
 ```

docs/source/examples/full-pipeline-web.md

@@ -258,14 +258,33 @@ The server starts at `http://localhost:8000`. The API docs are at `http://localh
 
 ### Docker Compose
 
+The FRAG workflow requires separately deployed RAG query and ingestion services.
+Set both endpoints to addresses that are reachable from the `aiq-agent`
+container. Container-local `localhost` points back to the AI-Q backend and is not
+a valid cross-service address.
+
+From the repository root:
+
 ```bash
-cd deploy
-cp .env.example .env
-# Edit .env with your API keys and set:
+cp deploy/.env.example deploy/.env
+# Edit deploy/.env with your API keys and these container-reachable values:
 # BACKEND_CONFIG=/app/configs/config_web_frag.yml
-docker compose up
+# RAG_SERVER_URL=http://rag-server:8081/v1
+# RAG_INGEST_URL=http://ingestor-server:8082/v1
+docker compose --env-file deploy/.env \
+  -f deploy/compose/docker-compose.yaml \
+  up -d --build --wait
 ```
 
+With the service-name endpoints shown above and both stacks running, connect
+the AI-Q backend to the RAG network:
+
+```bash
+docker network connect nvidia-rag aiq-agent
+```
+
+Repeat this command whenever the `aiq-agent` container is recreated.
+
 ### Test the Pipeline
 
 ```bash

docs/source/examples/skills-sandbox/index.md

@@ -228,10 +228,15 @@ Each skill should be a directory with a `SKILL.md` file:
 
 ```text
 src/aiq_agent/agents/deep_researcher/skills/
-`-- my-skill/
-    `-- SKILL.md
+`-- research/
+    `-- my-skill/
+        `-- SKILL.md
 ```
 
+The required hierarchy is `skills/<collection>/<skill>/SKILL.md`. The
+collection directory is the public name assigned to an agent in
+`deep_research_skills.agents`.
+
 At minimum, `SKILL.md` needs frontmatter with a stable `name` and a clear `description`:
 
 ```markdown
@@ -262,14 +267,18 @@ Skill descriptions matter because DeepAgents uses the frontmatter description to
 
 To add a built-in AI-Q deep research skill:
 
-1. Create a new directory under `src/aiq_agent/agents/deep_researcher/skills/`.
+1. Create `src/aiq_agent/agents/deep_researcher/skills/<collection>/<skill>/`.
 2. Add a `SKILL.md` file with frontmatter and workflow instructions.
 3. Put optional helper scripts, references, or templates inside the same skill directory.
 4. Reference any helper files from `SKILL.md` so the agent knows when to read or run them.
 5. Keep workflow instructions generic enough to handle variations of the task, but concrete enough to force required tool calls.
 6. Run with `configs/config_domain_routing_and_skills.yml` and test a query that should trigger the new skill.
 
-No config change is required for additional built-in skills inside an enabled collection. AI-Q collects available skill directories at runtime and exposes them to DeepAgents through an internal `/skills/` source.
+A skill added to a collection that is already assigned to the target agent needs
+no config change. For a new collection, add the collection name to the target
+agent under `deep_research_skills.agents`. AI-Q collects the assigned skill
+directories at runtime and exposes them to DeepAgents through an internal
+`/skills/` source.
 
 ## Notes and Limitations
 

docs/source/extending/adding-a-data-source.md

@@ -257,7 +257,7 @@ version = "1.0.0"
 description = "NAT-based patent search data source"
 requires-python = ">=3.11,<3.14"
 dependencies = [
-    "nvidia-nat==1.5.0",
+    "nvidia-nat-core==1.8.0",
     "httpx>=0.24.0",
     "pydantic>=2.0.0",
 ]
@@ -405,11 +405,14 @@ The LLM will use the parameter descriptions to decide which filters to apply.
 
 Format results so the agent can extract citations. Use a consistent pattern:
 
-```python
-# XML Document format (matches Tavily pattern)
-f'<Document href="{url}">\n<title>\n{title}\n</title>\n{content}\n</Document>'
+For pseudo-XML output, use the
+[fixed-shape renderer pattern](./adding-a-tool.md#output-formatting), which
+escapes every provider-controlled field before interpolation. Do not insert raw
+provider output into markup.
+
+For a plain structured-text alternative:
 
-# Or structured text format
+```python
 f"**{title}** ({id})\nAbstract: {abstract}\nLink: {url}"
 ```