ux: improve clarity and use of skills and sandbox in config

configs/config_cli_default.yml

@@ -128,6 +128,7 @@ functions:
 
   deep_research_agent:
     _type: deep_research_agent
+    enable_citation_verification: true
     orchestrator_llm: gpt_oss_llm
     source_router_llm: nemotron_super_llm
     researcher_llm: nemotron_super_llm

configs/config_domain_routing_and_skills.yml

@@ -172,6 +172,27 @@ functions:
     max_results: 5
     serper_api_key: ${SERPER_API_KEY}
 
+  deep_research_skills:
+    _type: deep_research_skills
+    agents:
+      researcher-agent: [research]
+      writer-agent: [synthesis]
+    require_sandbox:
+      - research
+
+  deep_research_sandbox:
+    _type: deep_research_sandbox
+    provider: modal
+    app_name: aiq-deep-research
+    image: python:3.13-slim
+    packages:
+      - matplotlib
+      - numpy
+      - pandas
+      - pillow
+      - tabulate  # required by pandas.DataFrame.to_markdown(), used by data-table-analysis skill
+    network: blocked
+
   # =========================================================================
   # Agents
   # =========================================================================
@@ -183,6 +204,7 @@ functions:
 
   deep_research_agent:
     _type: deep_research_agent
+    enable_citation_verification: false
     orchestrator_llm: gpt_oss_llm
     source_router_llm: nemotron_super_llm
     researcher_llm: nemotron_super_llm
@@ -193,26 +215,8 @@ functions:
       - web_search_tool
     verbose: true
     domain_catalog_path: configs/domain_catalogs/deep_research_domain_catalog.yml
-    skills:
-      enabled: true
-      agent_sources:
-        researcher:
-          - /skills/research-sandbox/
-        writer-agent:
-          - /skills/synthesis/
-      sandbox_required_sources:
-        - /skills/research-sandbox/
-    sandbox:
-      provider: modal
-      app_name: aiq-deep-research
-      image: python:3.13-slim
-      python_packages:
-        - matplotlib
-        - numpy
-        - pandas
-        - pillow
-        - tabulate  # required by pandas.DataFrame.to_markdown(), used by data-table-analysis skill
-      block_network: true
+    skills: deep_research_skills
+    sandbox: deep_research_sandbox
 
 workflow:
   _type: deep_research_workflow

configs/config_frontier_models.yml

@@ -148,6 +148,7 @@ functions:
 
   deep_research_agent:
     _type: deep_research_agent
+    enable_citation_verification: true
     orchestrator_llm: openai_gpt_5_2
     source_router_llm: nemotron_super_llm
     researcher_llm: nemotron_super_llm

configs/config_web_default_llamaindex.yml

@@ -202,6 +202,7 @@ functions:
 
   deep_research_agent:
     _type: deep_research_agent
+    enable_citation_verification: true
     orchestrator_llm: gpt_oss_llm
     source_router_llm: nemotron_super_llm
     researcher_llm: nemotron_super_llm

configs/config_web_frag.yml

@@ -170,6 +170,7 @@ functions:
 
   deep_research_agent:
     _type: deep_research_agent
+    enable_citation_verification: true
     orchestrator_llm: gpt_oss_llm
     source_router_llm: nemotron_super_llm
     researcher_llm: nemotron_super_llm

docs/source/architecture/agents/deep-researcher.md

@@ -141,6 +141,9 @@ Configured through `DeepResearchAgentConfig` (NeMo Agent Toolkit type name: `dee
 | `planner_llm` | `LLMRef` or `None` | `None` | LLM for planner subagent; falls back to `orchestrator_llm` if unset |
 | `writer_llm` | `LLMRef` or `None` | `None` | LLM for writer subagent; falls back to `orchestrator_llm` if unset |
 | `tools` | `list[FunctionRef \| FunctionGroupRef]` | `[]` | Research tools (web search, paper search, etc.) |
+| `skills` | `FunctionRef`, inline `deep_research_skills`, or `None` | `None` | Optional built-in skill collection assignments by agent name |
+| `sandbox` | `FunctionRef`, inline `deep_research_sandbox`, or `None` | `None` | Optional sandbox profile for DeepAgents `execute` support |
+| `enable_citation_verification` | `bool` | `true` | Verify generated citations against sources captured from tools |
 | `verbose` | `bool` | `true` | Enable detailed logging |
 
 **Example YAML:**

docs/source/examples/index.md

@@ -14,4 +14,4 @@ Complete, annotated configuration examples for common use cases.
 | [Full Pipeline -- Foundational RAG](./full-pipeline-web.md) | Complete production setup with hosted RAG | `config_web_frag.yml` |
 | [CLI with Local NIMs](./cli-with-local-nims.md) | Interactive CLI mode with self-hosted NIM models | `config_cli_default.yml` |
 | [Hybrid Frontier Model](./hybrid-frontier-model.md) | NIM for shallow + frontier model for deep research | Custom hybrid |
-| [Deep Research Skills and Sandbox](./skills-sandbox/index.md) | DeepAgents skills with Modal sandbox execution for quantitative research workflows | `config_skills.yml` |
+| [Deep Research Skills and Sandbox](./skills-sandbox/index.md) | DeepAgents skills with Modal sandbox execution for quantitative research workflows | `config_domain_routing_and_skills.yml` |

docs/source/examples/skills-sandbox/index.md

@@ -53,52 +53,54 @@ See Modal's token configuration docs for details: [modal.config](https://modal.c
 
 ## Configuration
 
-Use `configs/config_skills.yml`. The relevant section is:
+Use `configs/config_domain_routing_and_skills.yml`. The relevant section is:
 
 ```yaml
 functions:
+  deep_research_skills:
+    _type: deep_research_skills
+    agents:
+      researcher-agent:
+        - research
+      writer-agent:
+        - synthesis
+    require_sandbox:
+      - research
+
+  deep_research_sandbox:
+    _type: deep_research_sandbox
+    provider: modal
+    app_name: aiq-deep-research
+    image: python:3.12-slim
+    packages:
+      - matplotlib
+      - numpy
+      - pandas
+      - pillow
+    network: blocked
+
   deep_research_agent:
     _type: deep_research_agent
-    skills:
-      enabled: true
-      agent_sources:
-        orchestrator:
-          - /skills/
-        planner-agent:
-          - /skills/
-        researcher:
-          - /skills/
-        writer-agent:
-          - /skills/synthesis/
-      sandbox_required_sources:
-        - /skills/
-    sandbox:
-      provider: modal
-      app_name: aiq-deep-research
-      image: python:3.12-slim
-      python_packages:
-        - matplotlib
-        - numpy
-        - pandas
-        - pillow
-      block_network: true
+    enable_citation_verification: true
+    skills: deep_research_skills
+    sandbox: deep_research_sandbox
 ```
 
-When `skills.enabled` is true, AI-Q preloads the built-in skill files into the DeepAgents virtual filesystem and passes the configured per-agent skill sources directly to DeepAgents. When the sandbox block is present, DeepAgents `execute` calls run inside a job-scoped Modal sandbox.
+AI-Q validates the public skill collection names (`research`, `synthesis`) and resolves them to DeepAgents source paths internally. When skills are configured, AI-Q mounts the configured built-in skill collections into the DeepAgents virtual filesystem. When the sandbox ref is present, DeepAgents `execute` calls run inside a job-scoped Modal sandbox.
 
 ## Run AI-Q
 
 ```bash
 dotenv -f deploy/.env run .venv/bin/nat run \
-  --config_file configs/config_skills.yml \
+  --config_file configs/config_domain_routing_and_skills.yml \
   --input "Compare the top 10 publicly traded semiconductor companies by 2024 revenue. Build a markdown table with revenue, YoY growth, market cap, and gross margin. Then rank them and compute summary statistics. Use the data analysis tool for all calculations."
 ```
 
 For API or UI testing:
 
 ```bash
 dotenv -f deploy/.env run .venv/bin/nat serve \
-  --config_file configs/config_skills.yml \
+  --config_file configs/config_domain_routing_and_skills.yml \
   --host 0.0.0.0 \
   --port 8000
 ```
@@ -185,13 +187,13 @@ To add a built-in AI-Q deep research skill:
 3. Put optional helper scripts, references, or templates inside the same skill directory.
 4. Reference any helper files from `SKILL.md` so the agent knows when to read or run them.
 5. Keep workflow instructions generic enough to handle variations of the task, but concrete enough to force required tool calls.
-6. Run with `configs/config_skills.yml` and test a query that should trigger the new skill.
+6. Run with `configs/config_domain_routing_and_skills.yml` and test a query that should trigger the new skill.
 
-No config change is required for additional built-in skills in this directory when `skills.enabled: true` is set. AI-Q collects available skill directories at runtime and exposes them through the `/skills/` source.
+No config change is required for additional built-in skills inside an enabled collection. AI-Q collects available skill directories at runtime and exposes them to DeepAgents through an internal `/skills/` source.
 
 ## Notes and Limitations
 
 - The Modal sandbox is used for code execution. Text artifacts that need to survive for the report should be written through DeepAgents filesystem tools to `/shared/...`.
 - `/shared/` is a virtual DeepAgents filesystem path. Use `ls`, `read_file`, `write_file`, and `edit_file` for `/shared/`; do not inspect `/shared/` with shell commands through `execute`.
-- The sandbox is configured with `block_network: true`, so research should happen through AI-Q search tools, not from sandbox code.
+- The sandbox is configured with `network: blocked`, so research should happen through AI-Q search tools, not from sandbox code.
 - For the first release, sandbox lifecycle cleanup, persistence policy, quotas, and production capacity controls are tracked as follow-up work.

frontends/aiq_api/src/aiq_api/jobs/runner.py

@@ -340,6 +340,13 @@ async def run_agent_job(
 
         async with WorkflowBuilder.from_config(config=config) as builder:
             fn_config = builder.get_function_config(agent_config_name)
+            if getattr(fn_config, "type", None) == "deep_research_agent":
+                from aiq_agent.agents.deep_researcher.register import DeepResearchAgentConfig
+                from aiq_agent.agents.deep_researcher.register import resolve_deep_research_runtime_config
+
+                if isinstance(fn_config, DeepResearchAgentConfig):
+                    skills_config, sandbox_config = resolve_deep_research_runtime_config(fn_config, builder)
+                    fn_config = fn_config.model_copy(update={"skills": skills_config, "sandbox": sandbox_config})
 
             provider, llm = await _create_llm_provider(builder, fn_config)
 
@@ -592,19 +599,24 @@ def _create_agent_instance(
     1. llm_provider + tools pattern (DeepResearcherAgent style)
     2. llm + tools pattern (simpler agents)
     """
-    # Try async deep_researcher pattern with generic function config and job-scoped runtime state.
-    try:
+    from aiq_agent.agents.deep_researcher.register import DeepResearchAgentConfig
+
+    if isinstance(fn_config, DeepResearchAgentConfig):
         return agent_cls(
             llm_provider=llm_provider,
             tools=tools,
             verbose=verbose,
             callbacks=callbacks,
-            config=fn_config,
+            domain_catalog_path=fn_config.domain_catalog_path,
+            enable_source_router=fn_config.enable_source_router,
+            enable_citation_verification=fn_config.enable_citation_verification,
+            skills=fn_config.skills,
+            sandbox=fn_config.sandbox,
             job_id=job_id,
+            max_research_concurrency=fn_config.max_research_concurrency,
+            max_concurrent_source_tool_calls=fn_config.max_concurrent_source_tool_calls,
+            max_source_tool_batch_size=fn_config.max_source_tool_batch_size,
         )
-    except TypeError as exc:
-        if "unexpected keyword argument" not in str(exc):
-            raise
 
     # Try original deep_researcher pattern (llm_provider + tools + verbose)
     try:

frontends/aiq_api/src/aiq_api/routes/jobs.py

@@ -929,35 +929,21 @@ async def _cancel_dask_task(scheduler_address: str, job_id: str) -> bool:
         job_id: Job ID to cancel.
 
     Returns:
-        True if task was cancelled, False otherwise.
+        True if a Dask cancellation request was sent, False otherwise.
     """
     try:
         from distributed import Client
         from distributed import Future
-        from distributed import Variable
 
         async with Client(scheduler_address, asynchronous=True) as client:
-            var = Variable(name=job_id, client=client)
-            try:
-                # Short timeout: variable may be unset if worker hasn't started or job already finished.
-                future = await var.get(timeout=2)
-                if isinstance(future, Future):
-                    await client.cancel([future], asynchronous=True, force=True)
-                    logger.info("Cancelled Dask task for job %s", job_id)
-                    return True
-            except (TimeoutError, asyncio.CancelledError) as e:
-                logger.warning(
-                    "Could not get Dask future for job %s (variable not set or wait cancelled): %s",
-                    job_id,
-                    type(e).__name__,
-                )
-            except Exception as e:
-                logger.warning("Error getting Dask future for job %s: %s", job_id, e)
-            finally:
-                try:
-                    var.delete()
-                except (KeyError, RuntimeError):
-                    pass
+            # NAT JobStore submits job futures with key ``{job_id}-job``. Targeting
+            # the key directly avoids using Dask Variable.get as a maybe-exists
+            # check, which logs scheduler-side timeout errors when the variable is
+            # absent or slow to resolve.
+            future = Future(f"{job_id}-job", client)
+            await client.cancel([future], asynchronous=True, force=True)
+            logger.info("Sent cancellation request for Dask task %s", future.key)
+            return True
     except (ConnectionError, TimeoutError, OSError) as e:
         logger.warning("Failed to cancel Dask task for job %s: %s", job_id, e)
     except Exception as e:

frontends/aiq_api/tests/test_periodic_cleanup.py

@@ -18,6 +18,8 @@
 from __future__ import annotations
 
 import asyncio
+import sys
+import types
 from datetime import UTC
 from datetime import datetime
 from datetime import timedelta
@@ -472,3 +474,46 @@ async def test_noop_when_no_task(self):
 
         jobs_module._cleanup_task = None
         await jobs_module.stop_periodic_cleanup()  # should not raise
+
+
+class TestCancelDaskTask:
+    """Tests for cancelling submitted Dask jobs."""
+
+    @pytest.mark.asyncio
+    async def test_cancels_deterministic_future_key_without_variable_get(self, monkeypatch):
+        from aiq_api.routes.jobs import _cancel_dask_task
+
+        calls: dict[str, object] = {}
+
+        class FakeFuture:
+            def __init__(self, key, client):
+                self.key = key
+                self.client = client
+
+        class FakeClient:
+            def __init__(self, scheduler_address, asynchronous):
+                calls["scheduler_address"] = scheduler_address
+                calls["asynchronous"] = asynchronous
+
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, exc_type, exc, tb):
+                return None
+
+            async def cancel(self, futures, asynchronous, force):
+                calls["cancelled_keys"] = [future.key for future in futures]
+                calls["cancel_asynchronous"] = asynchronous
+                calls["force"] = force
+
+        fake_distributed = types.SimpleNamespace(Client=FakeClient, Future=FakeFuture)
+        monkeypatch.setitem(sys.modules, "distributed", fake_distributed)
+
+        assert await _cancel_dask_task("tcp://localhost:8786", "job-123") is True
+        assert calls == {
+            "scheduler_address": "tcp://localhost:8786",
+            "asynchronous": True,
+            "cancelled_keys": ["job-123-job"],
+            "cancel_asynchronous": True,
+            "force": True,
+        }

skills/aiq-deploy/references/configs.md

@@ -27,7 +27,7 @@ Use these repository docs as the source of truth:
 | `configs/config_web_frag.yml` | Foundational RAG / FRAG mode | Requires reachable `RAG_SERVER_URL` and `RAG_INGEST_URL`. Read `frag.md` before using. |
 | `configs/config_cli_default.yml` | Interactive terminal CLI mode | Not enough for `aiq-research`, because it does not provide the web/API backend expected by the skill. |
 | `configs/config_frontier_models.yml` | Hybrid model experiments | Advanced. May require additional provider keys or model access beyond the default NIM-backed path. |
-| `configs/config_skills.yml` | AI-Q runtime DeepAgents skills and sandbox behavior | Advanced. This is not the external Agent Skill packaging mechanism and should not be selected only because the user says "AI-Q as a skill." |
+| `configs/config_domain_routing_and_skills.yml` | AI-Q runtime DeepAgents skills and sandbox behavior | Advanced. This is not the external Agent Skill packaging mechanism and should not be selected only because the user says "AI-Q as a skill." |
 
 Default to `config_web_default_llamaindex.yml` unless the user explicitly chooses CLI, FRAG, or an advanced example.
 If no existing config matches the request, stop and explain the customization gap instead of inventing a config.