Separate native foundation and trading catalogs with upstream acceptance

.github/workflows/native-foundation-e2e.yml

@@ -0,0 +1,137 @@
+name: Native foundation offline engine E2E
+
+on:
+  push:
+    paths:
+      - '.github/workflows/native-foundation-e2e.yml'
+      - 'scripts/verify_nautilus_ci.py'
+      - 'tests/test_native_nautilus_ci.py'
+      - 'blueprints/us-equities/engine-nautilus/**'
+      - 'evidence/receipts/native-nautilus-v2-20260920.json'
+  pull_request:
+    paths:
+      - '.github/workflows/native-foundation-e2e.yml'
+      - 'scripts/verify_nautilus_ci.py'
+      - 'tests/test_native_nautilus_ci.py'
+      - 'blueprints/us-equities/engine-nautilus/**'
+      - 'evidence/receipts/native-nautilus-v2-20260920.json'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  nautilus-offline-replay:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out public acceptance sources
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Create an owned traversable root for public CI inputs and reports
+        run: |
+          umask 022
+          foundation_root=$(mktemp --directory /tmp/native-foundation-e2e.XXXXXX)
+          chmod 0755 -- "$foundation_root"
+          mkdir --mode=0755 -- "$foundation_root/reports"
+          printf 'NATIVE_FOUNDATION_ROOT=%s\n' "$foundation_root" >> "$GITHUB_ENV"
+      - name: Check independent verification failure modes
+        run: |
+          python3 -m unittest tests.test_native_nautilus_ci -v 2>&1 \
+            | tee "$NATIVE_FOUNDATION_ROOT/reports/checks.txt"
+      - name: Prepare native CPython and mandatory namespace isolation
+        run: |
+          umask 022
+          output="$NATIVE_FOUNDATION_ROOT/reports"
+          sudo apt-get update 2>&1 | tee "$output/apt-update.txt"
+          sudo apt-get install --yes --no-install-recommends bubblewrap python3.12-venv \
+            2>&1 | tee "$output/apt-install.txt"
+          python3.12 --version >"$output/python-version.txt"
+          bwrap --version >"$output/bubblewrap-version.txt"
+          ldd --version >"$output/glibc-version.txt"
+          readlink /proc/self/ns/net >"$output/host-network-namespace.txt"
+          python3.12 -m venv "$NATIVE_FOUNDATION_ROOT/venv"
+      - name: Install exact accepted distributions using upstream pip
+        run: |
+          umask 022
+          output="$NATIVE_FOUNDATION_ROOT/reports"
+          python="$NATIVE_FOUNDATION_ROOT/venv/bin/python"
+          packages=(nautilus_trader==2.0.0rc5 numpy==2.5.3 pandas==3.0.6 python-dateutil==2.9.0.post0 six==1.17.0)
+          "$python" -m pip --isolated download --disable-pip-version-check --no-input \
+            --index-url https://pypi.org/simple --only-binary=:all: --no-deps --pre \
+            --dest "$NATIVE_FOUNDATION_ROOT/wheels" "${packages[@]}" \
+            >"$output/download.stdout" 2>"$output/download.stderr"
+          printf '%s  %s\n' eab45fafd2312deda1236554c49a9798bfc76bc8465af864878e2f70189ebebe \
+            "$NATIVE_FOUNDATION_ROOT/wheels/nautilus_trader-2.0.0rc5-cp312-cp312-manylinux_2_34_x86_64.whl" \
+            | sha256sum --check >"$output/engine-wheel-check.txt"
+          sha256sum "$NATIVE_FOUNDATION_ROOT/wheels/"*.whl >"$output/downloaded-wheel-hashes.txt"
+          "$python" -m pip --isolated install --disable-pip-version-check --no-input \
+            --no-index --find-links "$NATIVE_FOUNDATION_ROOT/wheels" --no-deps --pre \
+            "${packages[@]}" >"$output/install.stdout" 2>"$output/install.stderr"
+          "$python" -m pip --isolated freeze >"$output/freeze.txt" 2>"$output/freeze.stderr"
+          "$python" -m pip --isolated check >"$output/pip-check.stdout" 2>"$output/pip-check.stderr"
+      - name: Fetch unchanged pinned upstream source and verify observer bytes
+        run: |
+          umask 022
+          output="$NATIVE_FOUNDATION_ROOT/reports"
+          curl --fail --silent --show-error --location --max-time 60 \
+            https://raw.githubusercontent.com/nautechsystems/nautilus_trader/1b0a49d2792a9432a3aca3fcb617ce7a630d905e/docs/getting_started/quickstart.py \
+            --output "$output/quickstart.py" >"$output/source-fetch.stdout" 2>"$output/source-fetch.stderr"
+          cp blueprints/us-equities/engine-nautilus/observe_quickstart.py "$output/observe_quickstart.py"
+          printf '%s  %s\n' \
+            487e6807dedd1a38062638eb671f6110799451611819542bf0f0c10646cb2c53 "$output/quickstart.py" \
+            b5cb94472619f6cd5432b7e410f4657fcf8db966fada9032c7dc188739565d30 "$output/observe_quickstart.py" \
+            | sha256sum --check >"$output/source-check.txt"
+      - name: Run the original quickstart twice without network or host credentials
+        run: |
+          umask 022
+          output="$NATIVE_FOUNDATION_ROOT/reports"
+          for run in run-1 run-2; do
+            run_output="$output/$run"
+            # Refuse existing paths: ownership changes apply only to this new run directory.
+            mkdir --mode=0755 -- "$run_output"
+            # Mandatory namespaces: failure is a failed acceptance, never a host-network fallback.
+            command=(timeout --signal=TERM --kill-after=5 120 sudo --non-interactive bwrap
+              --unshare-all --unshare-net --die-with-parent --new-session --clearenv
+              --uid "$(id -u)" --gid "$(id -g)" --cap-drop ALL
+              --ro-bind /usr /usr --symlink usr/bin /bin --symlink usr/lib /lib
+              --symlink usr/lib64 /lib64 --proc /proc --dev /dev --tmpfs /tmp
+              --ro-bind "$NATIVE_FOUNDATION_ROOT/venv" /venv
+              --ro-bind "$output/quickstart.py" /input/quickstart.py
+              --ro-bind "$output/observe_quickstart.py" /input/observe_quickstart.py
+              --bind "$run_output" /out --chdir /out
+              --setenv HOME /tmp --setenv LANG C.UTF-8 --setenv PATH /usr/bin:/bin --setenv PWD /out
+              --setenv PYTHONDONTWRITEBYTECODE 1 --setenv PYTHONHASHSEED 0
+              --setenv OPENBLAS_NUM_THREADS 1 --setenv OMP_NUM_THREADS 1
+              /venv/bin/python -I /input/observe_quickstart.py)
+            printf '%q ' "${command[@]}" >"$run_output/command.txt"
+            date --utc --iso-8601=seconds >"$run_output/started.txt"
+            touch "$run_output/stdout" "$run_output/stderr" "$run_output/exit-code.txt" "$run_output/finished.txt"
+            # sudo+bwrap maps the inside UID to outside root. Only /out needs that owner;
+            # the existing command logs stay runner-owned, and native reports remain readable.
+            sudo --non-interactive chown --no-dereference 0:0 -- "$run_output"
+            status=0
+            "${command[@]}" >"$run_output/stdout" 2>"$run_output/stderr" || status=$?
+            printf '%s\n' "$status" >"$run_output/exit-code.txt"
+            date --utc --iso-8601=seconds >"$run_output/finished.txt"
+            if [ "$status" -ne 0 ]; then
+              echo "Native offline acceptance failed; inspect retained $run stdout/stderr. No isolation fallback was attempted." >&2
+              exit "$status"
+            fi
+          done
+      - name: Independently verify isolation, cleanup, cash and exact economic replay
+        if: ${{ !cancelled() && env.NATIVE_FOUNDATION_ROOT != '' }}
+        run: python3 scripts/verify_nautilus_ci.py --output "$NATIVE_FOUNDATION_ROOT/reports"
+      - name: Retain original commands, outputs and verification failures
+        if: ${{ !cancelled() && env.NATIVE_FOUNDATION_ROOT != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: native-foundation-e2e-results
+          path: ${{ env.NATIVE_FOUNDATION_ROOT }}/reports/
+          if-no-files-found: error
+          include-hidden-files: false
+          retention-days: 14

.github/workflows/validate.yml

@@ -29,6 +29,8 @@ jobs:
         run: python3 scripts/validate.py
       - name: Validate catalog evidence and coverage joins
         run: python3 scripts/validate_catalogs.py
+      - name: Validate general foundation capability claims
+        run: python3 scripts/validate_foundation.py --root . --json
       - name: Validate declared convergence evidence and scope
         run: >-
           python3 scripts/validate_convergence.py

AGENTS.md

@@ -1,5 +1,12 @@
 # Repository work
 
+The two maintained catalogs start at `catalogs/README.md`. Use
+`catalogs/foundation/manifest.json` for general native harness layers and
+`catalogs/us-equities/README.md` for the separate trading architecture. Apply
+`docs/harness-defaults.md` when building or changing a harness: supported upstream
+installation, capability-specific evidence, bounded workers, scoped state and
+recoverable lifecycle acceptance. Load only the layer needed by the task.
+
 Carry authorized setup, fixes, checks and documentation through useful completion.
 Use a short internal plan; do not add intake, brainstorming or separate planning
 approval to bounded work. Reuse passing evidence when its inputs still match and

adoption/manifest.json

@@ -23,22 +23,22 @@
   "sources": {
     "components": "manifests/stack.json",
     "evidence": "manifests/evidence.json",
-    "open_gates": "catalogs/us-equities/convergence-review.json",
+    "open_gates": "catalogs/foundation/manifest.json",
     "instructions": "AGENTS.md",
     "boundaries": "docs/evidence.md",
     "current_activation": "observability/desktop-restart.md",
     "update_protocol": "adoption/update.md",
     "adoption_receipt": "adoption/receipt.json",
     "portability_research": "adoption/research.json",
     "current_paired_acceptance": "adoption/paired/receipt.json",
-    "current_convergence": "blueprints/us-equities/convergence-program/plan.json",
+    "current_convergence": "catalogs/foundation/manifest.json",
     "grand_dashboard": "observability/grand-dashboard/README.md",
     "current_historical_simulation": "blueprints/us-equities/historical-simulation/receipt.json",
     "current_data_readiness": "blueprints/us-equities/data-readiness/plan.json",
     "current_catalyst_dataset": "blueprints/us-equities/catalyst-dataset/native-receipt.json",
     "current_corporate_action_readiness": "blueprints/us-equities/corporate-action-readiness/receipt.json",
     "current_authenticated_data": "blueprints/us-equities/authenticated-data/native-receipt.json",
-    "current_wave": "blueprints/us-equities/catalyst-convergence/plan.json",
+    "current_wave": "catalogs/foundation/decisions.json",
     "current_identity_observation": "blueprints/us-equities/identity-readiness/native-receipt.json",
     "ecosystem_explorer": "docs/ecosystem/index.html",
     "ecosystem_explorer_guide": "docs/ecosystem/README.md",
@@ -71,7 +71,14 @@
     "wsl_native_quality": "blueprints/convergence-practice/wsl-native-tools/README.md",
     "current_selected_lifecycle": "adoption/lifecycle.md",
     "current_token_confirmation": "evidence/receipts/token-practice-confirmation-20260920.json",
-    "current_token_clean_prefix": "evidence/receipts/native-token-clean-prefix-20260920.json"
+    "current_token_clean_prefix": "evidence/receipts/native-token-clean-prefix-20260920.json",
+    "foundation_catalog": "catalogs/foundation/manifest.json",
+    "foundation_decisions": "catalogs/foundation/decisions.json",
+    "harness_defaults": "docs/harness-defaults.md",
+    "trading_target": "catalogs/us-equities/runtime-target.json",
+    "trading_open_gates": "catalogs/us-equities/convergence-review.json",
+    "trading_convergence": "blueprints/us-equities/convergence-program/plan.json",
+    "trading_wave": "blueprints/us-equities/catalyst-convergence/plan.json"
   },
   "recipe_map": {
     "omniroute": "docs/foundation-stack.md",
@@ -139,7 +146,9 @@
     "postgresql": "blueprints/convergence-practice/application-delivery/README.md",
     "poppler": "blueprints/convergence-practice/document-ingestion/README.md",
     "apple-container": "blueprints/convergence-practice/container-storage/README.md",
-    "playwright-test": "blueprints/convergence-practice/wsl-application/README.md"
+    "playwright-test": "blueprints/convergence-practice/wsl-application/README.md",
+    "tavily-cli": "recipes/tavily.md",
+    "nautilus-trader": "blueprints/us-equities/engine-nautilus/README.md"
   },
   "profiles": [
     {
@@ -282,6 +291,23 @@
         "blueprints/us-equities/state-recovery/README.md",
         "blueprints/us-equities/hosting/backup/README.md"
       ]
+    },
+    {
+      "id": "trading-nautilus",
+      "label": "Optional pinned Nautilus engine with separate broker boundaries",
+      "required_commands": [
+        "python3",
+        "uv",
+        "bwrap"
+      ],
+      "component_ids": [
+        "nautilus-trader",
+        "alpaca-py"
+      ],
+      "recipe_paths": [
+        "blueprints/us-equities/engine-nautilus/README.md",
+        "catalogs/us-equities/runtime-target.json"
+      ]
     }
   ],
   "toolchain": {
@@ -303,6 +329,9 @@
       "adoption/update.md"
     ],
     "read_on_demand": [
+      "catalogs/README.md",
+      "catalogs/foundation/README.md",
+      "docs/harness-defaults.md",
       "catalogs/us-equities/convergence-review.json",
       "docs/activation.md",
       "docs/evidence.md",
@@ -312,13 +341,17 @@
       "blueprints/us-equities/catalyst-convergence/README.md"
     ],
     "next_action_refs": [
+      "worker-cancellation-crash-resume",
+      "offhost-keys-and-service-reboot"
+    ],
+    "public_state_is_reference_only": true,
+    "trading_next_action_refs": [
       "research-specification",
       "accepted-pit-source",
       "retrieval-quality",
       "memory-lifecycle",
       "off-host-recovery"
-    ],
-    "public_state_is_reference_only": true
+    ]
   },
   "policy": {
     "historical_acceptance_transfers": false,