feat(deep-research): provider-neutral sandbox + durable artifact runtime

.gitignore

@@ -71,6 +71,12 @@ tmp/
 temp/
 *.tmp
 
+# Local sandbox artifact dumps (regeneratable, never committed)
+artifacts_out*/
+
+# OpenShell policy files generated by scripts/setup_openshell.sh
+configs/openshell/generated/
+
 
 # ============================================================================
 # Evaluation Dataset, Results and Outputs (Added for evaluation suite)

configs/config_openshell.yml

@@ -0,0 +1,176 @@
+# EXPERIMENTAL: AI-Q deep research over a local, single-operator OpenShell sandbox.
+# Run ./scripts/setup_openshell.sh first to provision the gateway and named sandbox.
+# Jobs attach to that shared sandbox; per-job directories prevent filename collisions
+# but are not a security boundary. Do not run mutually untrusted jobs concurrently.
+
+general:
+  use_uvloop: true
+  telemetry:
+    logging:
+      console:
+        _type: console
+        level: INFO
+
+  front_end:
+    _type: aiq_api
+    runner_class: aiq_api.plugin.AIQAPIWorker
+    db_url: ${NAT_JOB_STORE_DB_URL:-sqlite+aiosqlite:///./jobs.db}
+    expiry_seconds: 86400
+    cors:
+      allow_origin_regex: 'http://localhost(:\d+)?|http://127.0.0.1(:\d+)?'
+      allow_methods:
+        - GET
+        - POST
+        - DELETE
+        - OPTIONS
+      allow_headers:
+        - "*"
+      allow_credentials: true
+      expose_headers:
+        - "*"
+
+llms:
+  nemotron_llm_intent:
+    _type: nim
+    model_name: nvidia/nemotron-3-super-120b-a12b
+    base_url: "https://integrate.api.nvidia.com/v1"
+    temperature: 0.5
+    top_p: 0.9
+    max_tokens: 4096
+    num_retries: 5
+    chat_template_kwargs:
+      enable_thinking: true
+
+  nemotron_super_llm:
+    _type: nim
+    model_name: nvidia/nemotron-3-super-120b-a12b
+    base_url: "https://integrate.api.nvidia.com/v1"
+    temperature: 0.7
+    top_p: 0.7
+    max_tokens: 65536
+    num_retries: 5
+    chat_template_kwargs:
+      enable_thinking: true
+
+  gpt_oss_llm:
+    _type: nim
+    model_name: openai/gpt-oss-120b
+    base_url: https://integrate.api.nvidia.com/v1
+    temperature: 1.0
+    top_p: 1.0
+    max_tokens: 256000
+    api_key: ${NVIDIA_API_KEY}
+    max_retries: 10
+
+  summary_llm:
+    _type: nim
+    model_name: nvidia/nemotron-mini-4b-instruct
+    base_url: "https://integrate.api.nvidia.com/v1"
+    api_key: ${NVIDIA_API_KEY}
+    temperature: 0.3
+    max_tokens: 100
+
+functions:
+  data_sources:
+    _type: data_source_registry
+    sources:
+      - id: web_search
+        name: "Web Search"
+        description: "Search the web for real-time information."
+        tools:
+          - web_search_tool
+          - advanced_web_search_tool
+      - id: knowledge_layer
+        name: "Knowledge Base"
+        description: "Search uploaded documents and files."
+        tools:
+          - knowledge_search
+
+  web_search_tool:
+    _type: tavily_web_search
+    max_results: 5
+    max_content_length: 1000
+
+  advanced_web_search_tool:
+    _type: tavily_web_search
+    max_results: 2
+    advanced_search: true
+
+  knowledge_search:
+    _type: knowledge_retrieval
+    backend: llamaindex
+    collection_name: ${COLLECTION_NAME:-test_collection}
+    generate_summary: true
+    summary_model: summary_llm
+    summary_db: ${AIQ_SUMMARY_DB:-sqlite+aiosqlite:///./summaries.db}
+    top_k: 5
+    chroma_dir: ${AIQ_CHROMA_DIR:-/tmp/chroma_data}
+
+  intent_classifier:
+    _type: intent_classifier
+    llm: nemotron_llm_intent
+    verbose: true
+
+  clarifier_agent:
+    _type: clarifier_agent
+    llm: nemotron_super_llm
+    planner_llm: nemotron_super_llm
+    max_turns: 3
+    enable_plan_approval: true
+    log_response_max_chars: 2000
+    verbose: true
+
+  shallow_research_agent:
+    _type: shallow_research_agent
+    llm: nemotron_super_llm
+    exclude_tools:
+      - advanced_web_search_tool
+    verbose: true
+    max_llm_turns: 10
+    max_tool_iterations: 5
+
+  # OpenShell configuration: skills + sandbox
+  deep_research_skills:
+    _type: deep_research_skills
+    agents:
+      researcher-agent: [research]
+      writer-agent: [synthesis, research]
+    require_sandbox:
+      - research
+
+  deep_research_sandbox:
+    _type: deep_research_sandbox
+    provider: openshell
+    sandbox_name: ${AIQ_OPENSHELL_SANDBOX_NAME:-aiq-openshell-demo}
+    policy: ${AIQ_OPENSHELL_POLICY_FILE:-configs/openshell/generated/aiq-openshell-policy.yaml}
+    workdir: /sandbox
+    network: blocked
+    timeout: 1200
+    idle_timeout: 1800
+    delete_on_exit: false
+    artifact_capture:
+      enabled: true
+      max_file_bytes: 50000000
+      allow_extensions: [.png, .jpg, .jpeg, .webp, .csv, .json, .md, .ipynb, .pdf]
+
+  deep_research_agent:
+    _type: deep_research_agent
+    enable_citation_verification: true
+    orchestrator_llm: gpt_oss_llm
+    source_router_llm: nemotron_super_llm
+    researcher_llm: nemotron_super_llm
+    planner_llm: gpt_oss_llm
+    writer_llm: gpt_oss_llm
+    exclude_tools:
+      - web_search_tool
+    verbose: true
+    skills: deep_research_skills
+    sandbox: deep_research_sandbox
+
+workflow:
+  _type: chat_deepresearcher_agent
+  verbose: true
+  enable_escalation: true
+  enable_clarifier: true
+  use_async_deep_research: true
+  checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db}

configs/openshell/aiq-research-policy.yaml

@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Generated by scripts/setup_openshell.sh.
+
+version: 1
+
+filesystem_policy:
+  include_workdir: true
+  read_only:
+    - /usr
+    - /lib
+    - /etc
+    - /app
+    - /var/log
+    - /proc/self
+    - /dev/urandom
+  read_write:
+    - /sandbox
+    - /workspace
+    - /tmp
+    - /dev/null
+
+# best_effort lets the sandbox start on hosts without Landlock (e.g. Docker Desktop on
+# macOS), but there filesystem confinement is silently dropped. Acceptable only for the
+# local single-operator demo; production must use `hard_requirement` (fail closed).
+landlock:
+  compatibility: best_effort
+
+process:
+  run_as_user: sandbox
+  run_as_group: sandbox
+
+network_policies:
+  github:
+    name: github-readonly
+    endpoints:
+      - host: api.github.com
+        port: 443
+        protocol: rest
+        enforcement: enforce
+        access: read-only
+      - host: github.com
+        port: 443
+        protocol: rest
+        enforcement: enforce
+        access: read-only
+    binaries:
+      - { path: /usr/bin/curl }
+      - { path: /usr/local/bin/python3 }
+      - { path: /usr/local/bin/python }
+      - { path: /usr/local/bin/pip }
+      - { path: /usr/local/bin/pip3 }
+  nvidia:
+    name: nvidia-api-readonly
+    endpoints:
+      - host: integrate.api.nvidia.com
+        port: 443
+        protocol: rest
+        enforcement: enforce
+        access: read-only
+    binaries:
+      - { path: /usr/bin/curl }
+      - { path: /usr/local/bin/python3 }
+      - { path: /usr/local/bin/python }
+      - { path: /usr/local/bin/pip }
+      - { path: /usr/local/bin/pip3 }
+  tavily:
+    name: tavily-api-readonly
+    endpoints:
+      - host: api.tavily.com
+        port: 443
+        protocol: rest
+        enforcement: enforce
+        access: read-only
+    binaries:
+      - { path: /usr/bin/curl }
+      - { path: /usr/local/bin/python3 }
+      - { path: /usr/local/bin/python }
+      - { path: /usr/local/bin/pip }
+      - { path: /usr/local/bin/pip3 }
+  serper:
+    name: serper-api-readonly
+    endpoints:
+      - host: google.serper.dev
+        port: 443
+        protocol: rest
+        enforcement: enforce
+        access: read-only
+    binaries:
+      - { path: /usr/bin/curl }
+      - { path: /usr/local/bin/python3 }
+      - { path: /usr/local/bin/python }
+      - { path: /usr/local/bin/pip }
+      - { path: /usr/local/bin/pip3 }

deploy/openshell/Dockerfile.aiq-demo

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM python:3.13-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Optional OpenShell sandbox log verbosity. Defaults to `warn` (OpenShell's stock
+# sandbox level, so default behavior is unchanged). Rebuild with
+# `--build-arg OPENSHELL_SANDBOX_LOG_LEVEL=debug` (or `setup_openshell.sh
+# --sandbox-log-level debug`) to surface process/relay detail (`OCSF PROC:` etc.)
+# in the container logs (`/var/log/openshell.*.log`, `openshell logs <name>`).
+ARG OPENSHELL_SANDBOX_LOG_LEVEL=warn
+ENV RUST_LOG=${OPENSHELL_SANDBOX_LOG_LEVEL}
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        bash \
+        ca-certificates \
+        curl \
+        iproute2 \
+        iptables \
+        procps \
+    && python -m pip install --no-cache-dir numpy pandas matplotlib pillow tabulate requests \
+    && rm -rf /var/lib/apt/lists/* \
+    && groupadd -r sandbox \
+    && useradd -r -g sandbox -d /sandbox -s /bin/bash sandbox \
+    && mkdir -p /sandbox /workspace /tmp \
+    && chown -R sandbox:sandbox /sandbox /workspace
+
+WORKDIR /workspace
+USER sandbox
+ENTRYPOINT ["/bin/bash"]

docs/source/architecture/agents/sandbox.md

@@ -5,43 +5,52 @@ SPDX-License-Identifier: Apache-2.0
 
 # Deep Research Sandbox Notes
 
-Deep research can optionally run DeepAgents `execute` calls in a Modal sandbox.
-In this release, sandboxes are scoped to a single async job: the Modal sandbox
-name is the resolved job ID. This prevents unrelated jobs from sharing sandbox
-filesystem state when each request receives a unique job ID.
+Deep research can optionally run DeepAgents `execute` calls through a sandbox provider
+(Modal, OpenShell, or any registered provider). Modal creates a sandbox per job. The
+experimental OpenShell path attaches to a pre-created named sandbox shared by its jobs;
+job-scoped directories prevent filename collisions but are not a security boundary.
 
 The sandbox is an internal execution detail. There are no sandbox-specific API
 endpoints, and job-level auth remains responsible for submit, stream, status,
-cancel, state, and report access.
+cancel, state, and report access. The one user-visible surface is the artifact
+runtime (`.../job/{job_id}/artifacts`), which is also auth-scoped to the job.
+
+> **Developer reference:** the full architecture, provider contract, config schema,
+> artifact pipeline, and troubleshooting live next to the code in
+> [`src/aiq_agent/agents/deep_researcher/sandbox/README.md`](../../../../src/aiq_agent/agents/deep_researcher/sandbox/README.md).
 
 ## Current Behavior
 
-- One sandbox name is used per deep research job when sandboxing is enabled.
-- The sandbox name is the resolved job ID.
-- Different jobs produce different sandbox names.
+- Modal uses one sandbox per deep research job. OpenShell currently attaches jobs to
+  the configured shared sandbox name and is intended for local, single-operator testing.
 - Synchronous sandbox-enabled runs use an internal per-agent runtime ID.
-- Job IDs must be valid Modal object names: 64 characters or fewer, using only
-  alphanumeric characters, dashes, periods, and underscores.
-- Modal `timeout` and `idle_timeout` control sandbox lifetime.
-- Files written inside the Modal workdir are temporary scratch state.
-- Durable results should be returned by the agent or written through DeepAgents
-  virtual filesystem paths such as `/shared/`.
+- Providers are selected by config (`sandbox.provider` + `providers.<name>`); the
+  provider is validated against the registry and gated by its declared capabilities.
+  OpenShell policy is provisioned externally and is not verified when AI-Q attaches.
+- Job IDs must satisfy each provider's object-name rules (Modal: 64 chars or fewer,
+  alphanumeric plus dash/period/underscore).
+- `timeout` bounds individual execution. Other lifecycle controls are provider-dependent.
+- Files written inside the workdir are temporary scratch state. Durable text should
+  be written through DeepAgents virtual paths such as `/shared/`; durable binaries
+  (charts, CSVs) are captured by the artifact runtime.
 
 ## Operational Notes
 
-- High concurrency creates one Modal sandbox per concurrent sandbox-enabled job.
-- If clients provide custom job IDs, they must not reuse a job ID for a new job.
-  Reuse can attach the job to an existing Modal sandbox until Modal terminates it.
-- Cancelled or failed jobs may leave sandbox scratch files until Modal terminates
-  the sandbox according to timeout settings.
-- If Modal removes a container mid-job, the job may fail and should be retried.
+- High-concurrency Modal runs create one sandbox per job. OpenShell runs share the named
+  sandbox and must not be used concurrently for mutually untrusted jobs. Optional submit-path
+  caps (`AIQ_MAX_SANDBOXES_PER_PRINCIPAL` / `AIQ_MAX_SANDBOXES_GLOBAL`, default-off) bound
+  concurrency/cost but do not provide filesystem isolation.
+- Custom client-supplied job IDs must not be reused for a new job.
+- The runtime closes provider sessions on success, failure, cancellation, and timeout.
+  A named OpenShell sandbox persists when `delete_on_exit` is disabled.
 
-## Deferred Hardening
+## Current Safeguards
 
-Planned follow-up work for production deployments:
+The following safeguards are in place:
 
-- Explicit sandbox cleanup on job success, failure, cancellation, and timeout.
-- Retry-on-stale-container handling for Modal `NotFoundError`.
-- Artifact capture rules for generated charts and binary outputs before cleanup.
-- Sandbox quota and concurrency controls.
-- Metrics and structured logs for sandbox create, reuse, failure, and cleanup.
+- Explicit sandbox cleanup on success, failure, cancellation, and timeout.
+- Idempotency-gated retry-on-stale-container handling.
+- Artifact capture for generated charts/binaries (validate -> store -> serve/embed),
+  with MIME-from-bytes spoof rejection, SVG sanitization, and an inline-render allowlist.
+- Sandbox quota and concurrency controls, and artifact retention via job-expiry cleanup.
+- Structured lifecycle logging for sandbox create, reuse, failure, and cleanup.