seathatflowsinourveins · Sep 27, 2026 · Sep 27, 2026 · Sep 27, 2026 · Sep 27, 2026 · Sep 28, 2026
@@ -138,6 +138,17 @@
       "path": "blueprints/us-equities/mover-v3/data-tools/requirements.lock",
       "parser": "requirements.txt"
     },
+    {
+      "path": "blueprints/runtime-workers/crawl4ai/requirements.in",
+      "parser": "requirements.txt"
+    },
+    {
+      "path": "blueprints/runtime-workers/crawl4ai/requirements.lock",
+      "parser": "requirements.txt"
+    },
+    {
+      "path": "blueprints/runtime-workers/crawl4ai/grader/package-lock.json"
+    },
     {
       "path": "blueprints/us-equities/mover-v3/study/runtime-requirements.txt",
       "parser": "requirements.txt"
@@ -198,6 +209,10 @@
     {
       "path": "evidence/artifacts/macos-syn-e2e-20260924/requirements-macos-arm64-py312.in",
       "lockfile": "evidence/artifacts/macos-syn-e2e-20260924/requirements-macos-arm64-py312.lock"
+    },
+    {
+      "path": "blueprints/runtime-workers/crawl4ai/grader/package.json",
+      "lockfile": "blueprints/runtime-workers/crawl4ai/grader/package-lock.json"
     }
   ],
   "excluded": [

@@ -0,0 +1,2 @@
+.research/
+__pycache__/
@@ -0,0 +1,26 @@
+{
+  "playwright": "1.63.0",
+  "chromium_revision": "1243",
+  "browser_version": "153.0.8010.12",
+  "install_args": [
+    "install",
+    "--only-shell",
+    "chromium"
+  ],
+  "artifacts": [
+    {
+      "url": "https://cdn.playwright.dev/builds/cft/153.0.8010.12/linux64/chrome-headless-shell-linux64.zip",
+      "mirror_path": "builds/cft/153.0.8010.12/linux64/chrome-headless-shell-linux64.zip",
+      "sha256": "a9da028861a0cf789ff25c2fed45f5f1aaf969ed9247835b6a7821a4f7af9d1d",
+      "size": 119809080,
+      "provenance": "SHA256 computed over HTTPS bytes at recipe construction; Playwright does not publish this checksum in browsers.json"
+    },
+    {
+      "url": "https://cdn.playwright.dev/builds/ffmpeg/1011/ffmpeg-linux.zip",
+      "mirror_path": "builds/ffmpeg/1011/ffmpeg-linux.zip",
+      "sha256": "ebc74fc5b94830176a3c2914ae96bd8bc7f6a91f4f33890230f84a172ee61ccc",
+      "size": 2376500,
+      "provenance": "SHA256 computed over HTTPS bytes at recipe construction; Playwright does not publish this checksum in browsers.json"
+    }
+  ]
+}
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# This file sets task-specific variables; it never replaces HOME or CODEX_HOME.
+set -euo pipefail
+umask 077
+export NAS_CRAWL4AI_PREFIX="$HOME/.local/share/codex-ecosystem/tools/crawl4ai-0.9.4"
+export NAS_CRAWL4AI_STATE="$HOME/.local/state/native-agent-stack/runtime-workers/crawl4ai"
+export NAS_CRAWL4AI_GRADER="$NAS_CRAWL4AI_PREFIX/grader/node_modules/.bin/promptfoo"
+export PYTHONDONTWRITEBYTECODE=1
+export XDG_CACHE_HOME="$NAS_CRAWL4AI_STATE/cache"
+export CRAWL4_AI_BASE_DIRECTORY="$NAS_CRAWL4AI_STATE/crawl"
+export PLAYWRIGHT_BROWSERS_PATH="$NAS_CRAWL4AI_PREFIX/venv/.cache/ms-playwright"
+export NLTK_DATA="$NAS_CRAWL4AI_STATE/cache/nltk"
+export HF_HOME="$NAS_CRAWL4AI_STATE/cache/huggingface"
+export TIKTOKEN_CACHE_DIR="$NAS_CRAWL4AI_STATE/cache/tiktoken"
+export LITELLM_LOCAL_MODEL_COST_MAP=true
+export DO_NOT_TRACK=1
+export UV_PYTHON_DOWNLOADS=never
@@ -0,0 +1,65 @@
+# Sources: upstream v0.9.4 Dockerfile, entrypoint.sh, supervisord.conf, config.yml.
+services:
+  crawl4ai:
+    container_name: rw-crawl4ai-${NAS_CRAWL4AI_STORE:?}
+    labels:
+      com.native-agent-stack.owner: gpt6-omniroute-framework-integration
+    image: unclecode/crawl4ai@sha256:048848e548fad60c670bd656cbb3eb204fd999709a30365d3697c411ce50796d
+    platform: linux/amd64
+    # Container root maps to the unprivileged host user. Native supervisor drops
+    # Redis/Gunicorn to appuser after reading host-owned 0600 secret mounts.
+    user: "0:0"
+    cap_drop: [ALL]
+    # Only bootstrap ownership and supervisor's native appuser transition.
+    cap_add: [CHOWN, DAC_OVERRIDE, SETUID, SETGID]
+    security_opt: ["no-new-privileges:true"]
+    read_only: true
+    tmpfs:
+      - /tmp:rw,nosuid,nodev,noexec,mode=1777
+    pull_policy: never
+    init: true
+    restart: "no"
+    shm_size: "1gb"
+    mem_limit: "3g"
+    cpus: 2
+    pids_limit: 512
+    ports:
+      - "127.0.0.1:${NAS_CRAWL4AI_PORT:?private host port required}:11235"
+    environment:
+      LLM_PROVIDER: "openai/${CRAWL4AI_MODEL:?model supplied by config}"
+      LLM_API_KEY: "local-loopback"
+      OPENAI_BASE_URL: "${CRAWL4AI_CONTAINER_BASE_URL:?validated selected arm}"
+      LLM_BASE_URL: "${CRAWL4AI_CONTAINER_BASE_URL:?validated selected arm}"
+      PYTHONDONTWRITEBYTECODE: "1"
+      CRAWL4AI_HOOKS_ENABLED: "false"
+      CRAWL4AI_ALLOW_INTERNAL_URLS: "${NAS_CRAWL4AI_INTERNAL_URLS:-false}"
+      CRAWL4_AI_BASE_DIRECTORY: "/var/lib/crawl4ai"
+      XDG_CACHE_HOME: "/var/cache/crawl4ai"
+      TMPDIR: "/var/cache/crawl4ai/tmp"
+      NLTK_DATA: "/var/cache/crawl4ai/nltk"
+      HF_HOME: "/var/cache/crawl4ai/huggingface"
+      TIKTOKEN_CACHE_DIR: "/var/cache/crawl4ai/tiktoken"
+      LITELLM_LOCAL_MODEL_COST_MAP: "true"
+      DO_NOT_TRACK: "1"
+    entrypoint: ["/bin/bash", "/app/recipe-entrypoint.sh"]
+    volumes:
+      - "${NAS_CRAWL4AI_PREFIX:?}/recipe/container-entrypoint.sh:/app/recipe-entrypoint.sh:ro"
+      - "${NAS_CRAWL4AI_PREFIX:?}/recipe/config/supervisord.conf:/app/supervisord.conf:ro"
+      - "${NAS_CRAWL4AI_PREFIX:?}/recipe/config/server.yml:/app/config.yml:ro"
+      - "${NAS_CRAWL4AI_PREFIX:?}/recipe/config/logging.ini:/app/recipe-logging.ini:ro"
+      - "${NAS_CRAWL4AI_STATE:?}/secrets/api_token:/run/secrets/api_token:ro"
+      - "${NAS_CRAWL4AI_STATE:?}/secrets/secret_key:/run/secrets/secret_key:ro"
+      - "${NAS_CRAWL4AI_STATE:?}/secrets/redis_password:/run/secrets/redis_password:ro"
+      - "${NAS_CRAWL4AI_STATE:?}/containers/${NAS_CRAWL4AI_STORE:?}/crawl:/var/lib/crawl4ai"
+      - "${NAS_CRAWL4AI_STATE:?}/containers/${NAS_CRAWL4AI_STORE:?}/cache:/var/cache/crawl4ai"
+      - "${NAS_CRAWL4AI_STATE:?}/containers/${NAS_CRAWL4AI_STORE:?}/redis:/var/lib/redis"
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "2"
+networks:
+  default:
+    name: rw-crawl4ai-${NAS_CRAWL4AI_STORE:?}-net
+    labels:
+      com.native-agent-stack.owner: gpt6-omniroute-framework-integration
@@ -0,0 +1,10 @@
+{
+  "docker": "${HOME}/.local/share/codex-ecosystem/tools/docker-rootless-29.8.1/bin/docker",
+  "docker_context": "rootless",
+  "python": "/usr/bin/python3.12",
+  "api_port": 3730,
+  "fixture_port": 3731,
+  "mirror_port": 3732,
+  "e2e_api_port": 3734,
+  "working_directory": "${HOME}/.local/state/native-agent-stack/runtime-workers/crawl4ai/work"
+}
@@ -0,0 +1,28 @@
+# Gunicorn logconfig uses stdlib logging.config.fileConfig.
+# INFO observes MCP request types without DEBUG's complete request payloads.
+[loggers]
+keys=root,gunicorn.error,gunicorn.access
+[handlers]
+keys=console
+[formatters]
+keys=plain
+[logger_root]
+level=INFO
+handlers=console
+[logger_gunicorn.error]
+level=INFO
+handlers=console
+propagate=0
+qualname=gunicorn.error
+[logger_gunicorn.access]
+level=INFO
+handlers=console
+propagate=0
+qualname=gunicorn.access
+[handler_console]
+class=StreamHandler
+level=INFO
+formatter=plain
+args=(sys.stderr,)
+[formatter_plain]
+format=%(levelname)s %(name)s %(message)s
@@ -0,0 +1,43 @@
+{
+  "role": "mcp-server",
+  "loaded_servers": [],
+  "limitation": "No external MCP client or SKILL.md loader found in the reviewed v0.9.4 sources; native client verification pending. These are consumer requirements, not loaded Crawl4AI configuration.",
+  "sources": ["adoption/templates/codex.config.template.toml", "adoption/templates/codex.stack-worker.config.toml", "adoption/skills/manifest.json"],
+  "consumer_limits": {
+    "context-mode": {
+      "command": "${ECO_ROOT}/bin/node",
+      "args": ["${ECO_ROOT}/tools/context-mode-1.0.169/lib/node_modules/context-mode/start.mjs"],
+      "disabled_tools": ["ctx_upgrade", "ctx_purge"],
+      "project": "consumer worker working directory; bind CONTEXT_MODE_PROJECT_DIR there"
+    },
+    "ai-memory": {
+      "url": "http://${AI_MEMORY_URL}/mcp",
+      "enabled_tools": ["memory_query", "memory_read_page", "memory_recent", "memory_status", "memory_briefing"]
+    },
+    "socraticode": {
+      "command": "${ECO_ROOT}/bin/node",
+      "args": ["${ECO_ROOT}/tools/socraticode-1.14.0/lib/node_modules/socraticode/dist/index.js"],
+      "enabled_tools": ["codebase_search", "codebase_status", "codebase_list_projects", "codebase_health"],
+      "env": {"SOCRATICODE_WATCHER": "manual"}
+    },
+    "qmd": {
+      "command": "${ECO_ROOT}/bin/qmd",
+      "args": ["--index", "native-agent-stack-catalog", "mcp"],
+      "collections": ["foundation-docs", "foundation-adoption", "us-equities-foundation", "us-equities-catalog"],
+      "scope": "consumer supplies explicit collection filters; this template does not configure a server-side ACL"
+    },
+    "headroom": {
+      "command": "${ECO_ROOT}/bin/headroom",
+      "args": ["mcp", "serve", "--proxy-url", "http://127.0.0.1:1"],
+      "enabled_tools": ["headroom_compress", "headroom_retrieve", "headroom_stats"],
+      "condition": "only consumers with large outputs not already contained by fit_markdown"
+    },
+    "serena": {"condition": "coding consumers only; not loaded by this crawler"},
+    "jcodemunch": {"condition": "coding consumers only; no launch entry in the two specified templates; do not invent one"}
+  },
+  "offered_endpoints": ["/mcp/sse", "/mcp/ws"],
+  "recommended_consumer_tools": ["md", "crawl"],
+  "container_llm_contract": "not qualified: untrusted /crawl bodies reject LLM strategies; built-in /llm and md:f=llm do not expose extra_args for the required temperature omission and per-call headers",
+  "consumer_request_constraints": {"md": {"f": ["fit", "raw", "bm25"]}, "crawl": "non-LLM extraction only"},
+  "authentication": "Authorization: Bearer read privately from the installed secrets/api_token file; never put it in a URL or committed config"
+}
@@ -0,0 +1,12 @@
+# Source: unclecode/crawl4ai v0.9.4 deploy/docker/config.yml + utils.py:114-138.
+# Native loader deep-merges this with upstream defaults. Provider/URL use env.
+app:
+  host: "127.0.0.1"
+  port: 11235
+  workers: 1
+security:
+  enabled: true
+  jwt_enabled: true
+  cors_allow_origins: []
+llm:
+  allowed_providers: ["openai"]
@@ -0,0 +1,34 @@
+[supervisord]
+nodaemon=true                   ; Run supervisord in the foreground
+logfile=/dev/null               ; Log supervisord output to stdout/stderr
+logfile_maxbytes=0
+
+[program:redis]
+; Loopback-only and password-protected. REDIS_PASSWORD is exported by
+; entrypoint.sh from the mounted secret; the app must supply it to connect.
+command=/usr/bin/redis-server --loglevel notice --bind 127.0.0.1 -::1 --requirepass "%(ENV_REDIS_PASSWORD)s" --dir /var/lib/redis
+user=appuser                    ; Run redis as our non-root user
+autorestart=true
+priority=10
+stdout_logfile=/dev/stdout      ; Redirect redis stdout to container stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr      ; Redirect redis stderr to container stderr
+stderr_logfile_maxbytes=0
+
+[program:gunicorn]
+; Bind is resolved by entrypoint.sh into GUNICORN_BIND: loopback unless a
+; credential (CRAWL4AI_API_TOKEN) is present, in which case the operator may
+; expose an explicitly selected interface. gunicorn is the socket guard; the Python
+; _resolve_auth() check is the in-process guard - both must agree.
+command=/usr/local/bin/gunicorn --bind "%(ENV_GUNICORN_BIND)s" --bind "127.0.0.1:11235" --workers 1 --threads 4 --timeout 1800 --graceful-timeout 30 --keep-alive 300 --log-level info --log-config /app/recipe-logging.ini --access-logfile - --limit-request-line 8190 --limit-request-fields 100 --worker-class uvicorn.workers.UvicornWorker server:app
+directory=/app                  ; Working directory for the app
+user=appuser                    ; Run gunicorn as our non-root user
+autorestart=true
+priority=20
+environment=PYTHONUNBUFFERED=1  ; Ensure Python output is sent straight to logs
+stdout_logfile=/dev/stdout      ; Redirect gunicorn stdout to container stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr      ; Redirect gunicorn stderr to container stderr
+stderr_logfile_maxbytes=0
+
+# Optional: Add filebeat or other logging agents here if needed
@@ -0,0 +1,64 @@
+{
+  "llm": {
+    "provider_prefix": "openai/",
+    "api_token": "local-loopback",
+    "max_tokens": 3000,
+    "timeout_seconds": 180
+  },
+  "content": {
+    "input_format": "fit_markdown",
+    "pruning_threshold": 0.35,
+    "pruning_threshold_type": "fixed",
+    "min_word_threshold": 1,
+    "chunk_token_threshold": 4096,
+    "overlap_rate": 0.0,
+    "word_token_rate": 1.3,
+    "max_fit_characters": 6000,
+    "excluded_tags": [
+      "nav",
+      "footer",
+      "aside"
+    ]
+  },
+  "e2e": {
+    "arms": [
+      "control",
+      "engines-on"
+    ],
+    "pages": [
+      "beacon.html",
+      "flask.html",
+      "lamp.html"
+    ],
+    "scope": "synthetic frozen fixture; native extraction plus container API/MCP transport checks"
+  },
+  "arms": {
+    "control": {
+      "model": "cx/gpt-6-astra-max",
+      "model_env": "CRAWL4AI_MODEL",
+      "base_url": "http://127.0.0.1:20128/v1",
+      "base_url_env": "CRAWL4AI_BASE_URL",
+      "container_base_url": "http://10.0.2.2:20128/v1",
+      "expected_effort": "max",
+      "gateway": "omniroute"
+    },
+    "engines-on": {
+      "model": "sharedgw/gpt-6-astra-max",
+      "model_env": "CRAWL4AI_MODEL",
+      "base_url": "http://127.0.0.1:20129/v1",
+      "base_url_env": "CRAWL4AI_BASE_URL",
+      "container_base_url": "http://10.0.2.2:20129/v1",
+      "expected_effort": "max",
+      "gateway": "omniroute-fw"
+    },
+    "comparison": {
+      "model": "cx/gpt-6-sol",
+      "model_env": "CRAWL4AI_COMPARISON_MODEL",
+      "base_url": "http://127.0.0.1:20128/v1",
+      "base_url_env": "CRAWL4AI_BASE_URL",
+      "container_base_url": "http://10.0.2.2:20128/v1",
+      "expected_effort": "medium",
+      "gateway": "omniroute"
+    }
+  }
+}
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# v0.9.4 entrypoint.sh:20-39; native supervisor/Gunicorn accepts two --binds.
+set -euo pipefail
+export SECRET_KEY="$(cat /run/secrets/secret_key)"
+export PLAYWRIGHT_BROWSERS_PATH="$(python3 -c 'import pwd; from pathlib import Path; print(Path(pwd.getpwnam("appuser").pw_dir) / ".cache/ms-playwright")')"
+# Only these dedicated worker mounts are initialized. Baked browser files stay
+# at their upstream image path. Dockerfile:180-191,204-209 and supervisor user=appuser.
+mkdir -p /var/lib/crawl4ai /var/lib/redis /var/cache/crawl4ai/tmp
+# CAP_CHOWN permits reclaiming only these mount roots before chmod; dropping
+# CAP_FOWNER means chmod must happen while bootstrap root owns the directory.
+chown 0:0 /var/lib/crawl4ai /var/lib/redis /var/cache/crawl4ai /var/cache/crawl4ai/tmp
+chmod 0700 /var/lib/crawl4ai /var/lib/redis /var/cache/crawl4ai /var/cache/crawl4ai/tmp
+chown appuser:appuser /var/lib/crawl4ai /var/lib/redis /var/cache/crawl4ai /var/cache/crawl4ai/tmp
+# Explicit interface + a second loopback bind in supervisord.conf for MCP's proxy.
+NAS_CRAWL4AI_IP="$(python3 -c 'import socket; print(socket.gethostbyname(socket.gethostname()))')"
+export GUNICORN_BIND="${NAS_CRAWL4AI_IP}:11235"
+cd /app
+exec /bin/bash /app/entrypoint.sh