From b136ad97206d163c24c363d9f721b8368a475cf9 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 01:33:16 +0300 Subject: [PATCH 01/22] ci(test): enforce hermetic deterministic suite --- .github/workflows/README.md | 7 + .github/workflows/code_style.yml | 1 + .github/workflows/reborn-e2e.yml | 28 ++- .github/workflows/reborn-tests.yml | 27 ++- docs/internal/hermetic-deterministic-suite.md | 53 +++++ scripts/ci/hermetic-network-guard.c | 206 +++++++++++++++++ scripts/ci/hermetic-network-probe.c | 46 ++++ scripts/ci/hermetic-network-runner.sh | 33 +++ .../ci/run-hermetic-deterministic-suite.sh | 169 ++++++++++++++ scripts/ci/run-hermetic-test-process.sh | 159 +++++++++++++ scripts/ci/test-hermetic-test-process.sh | 215 ++++++++++++++++++ tests/e2e/conftest.py | 4 +- tests/e2e/hermetic_process.py | 36 +++ tests/e2e/reborn_webui_harness.py | 2 + .../scenarios/test_reborn_responses_api.py | 2 + 15 files changed, 974 insertions(+), 14 deletions(-) create mode 100644 docs/internal/hermetic-deterministic-suite.md create mode 100644 scripts/ci/hermetic-network-guard.c create mode 100644 scripts/ci/hermetic-network-probe.c create mode 100755 scripts/ci/hermetic-network-runner.sh create mode 100755 scripts/ci/run-hermetic-deterministic-suite.sh create mode 100755 scripts/ci/run-hermetic-test-process.sh create mode 100755 scripts/ci/test-hermetic-test-process.sh create mode 100644 tests/e2e/hermetic_process.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3bff86e3e7a..894bb29eb58 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -18,6 +18,13 @@ on `push` to main, the merge queue must run it in the same shape first instead). External/live checks (canaries, deploys, releases, benchmark thresholds) are exempt: they stay out of the queue by design. +The canonical local composition of the deterministic Reborn gates is +`scripts/ci/run-hermetic-deterministic-suite.sh all`. CI invokes the same +checked-in stages through that runner so credentials, ambient behavior, +mutable roots, clock/seed inputs, and non-loopback egress have one mechanical +boundary. Setup and exclusions are documented in +`docs/internal/hermetic-deterministic-suite.md`. + The WASM WIT compatibility lane uses two risk scopes. Pull requests run it only for direct WIT, WASM host, extension, compatibility-test, or lane-workflow changes. Root `Cargo.toml` and `Cargo.lock` changes are broader workspace risk: diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 31978244929..dfb469eb494 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -233,6 +233,7 @@ jobs: scripts/ci/test-reborn-crate-test-buckets.sh python3 scripts/ci/test_ws12_suite_shards.py python3 scripts/ci/test_ws12_workflow_contracts.py + scripts/ci/test-hermetic-test-process.sh bash .github/scripts/test-pr-labeler.sh clippy: diff --git a/.github/workflows/reborn-e2e.yml b/.github/workflows/reborn-e2e.yml index ea2c7dd6dce..a49d3763b95 100644 --- a/.github/workflows/reborn-e2e.yml +++ b/.github/workflows/reborn-e2e.yml @@ -50,6 +50,7 @@ on: - "crates/ironclaw_*/**" - "docs/reborn/**" - "scripts/reborn-e2e-rust.sh" + - "scripts/ci/*hermetic*" - "tests/e2e/**" - "build.rs" - "providers.json" @@ -99,7 +100,7 @@ jobs: # Keep this pattern in sync with the push `paths:` filter in the # `on:` block above. - if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(crates/ironclaw_[^/]+/|docs/reborn/|scripts/reborn-e2e-rust\.sh$|tests/e2e/|build\.rs$|providers\.json$|Cargo\.toml$|Cargo\.lock$|\.github/workflows/reborn-e2e\.yml$)'; then + if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(crates/ironclaw_[^/]+/|docs/reborn/|scripts/reborn-e2e-rust\.sh$|scripts/ci/[^/]*hermetic[^/]*$|tests/e2e/|build\.rs$|providers\.json$|Cargo\.toml$|Cargo\.lock$|\.github/workflows/reborn-e2e\.yml$)'; then echo "has_e2e_scope=true" >> "$GITHUB_OUTPUT" else echo "has_e2e_scope=false" >> "$GITHUB_OUTPUT" @@ -138,7 +139,10 @@ jobs: - name: Run deterministic Reborn Rust gate env: CARGO_TEST_ARGS: "-- --nocapture" - run: scripts/reborn-e2e-rust.sh ${{ matrix.group }} + run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + rust-e2e + ${{ matrix.group }} webui-v2-smoke: name: Reborn WebUI v2 smoke @@ -276,6 +280,8 @@ jobs: - name: Run Reborn WebUI v2 smoke run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command pytest tests/e2e/scenarios/test_reborn_webui_v2_smoke.py tests/e2e/scenarios/test_reborn_webui_v2_sso.py @@ -286,6 +292,8 @@ jobs: env: IRONCLAW_EMULATE_CLI: ${{ github.workspace }}/.emulate/packages/emulate/dist/index.js run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command pytest tests/e2e/scenarios/test_provider_capability_inventory.py tests/e2e/scenarios/test_journey_coverage.py @@ -306,6 +314,8 @@ jobs: IRONCLAW_EMULATE_CLI: ${{ github.workspace }}/.emulate/packages/emulate/dist/index.js IRONCLAW_JOURNEY_ORDER: reverse run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command pytest tests/e2e/scenarios/test_reborn_qa_trace_full_path.py::test_mutating_qa_journeys_replay_in_reverse_against_shared_provider_world -v @@ -331,7 +341,8 @@ jobs: echo "replaying ${#journeys[@]} journeys alone" for journey in "${journeys[@]}"; do echo "::group::alone: $journey" - pytest tests/e2e/scenarios/test_reborn_qa_trace_full_path.py \ + scripts/ci/run-hermetic-deterministic-suite.sh command \ + pytest tests/e2e/scenarios/test_reborn_qa_trace_full_path.py \ -k "$journey and not isolated" \ -v --timeout=120 echo "::endgroup::" @@ -346,7 +357,8 @@ jobs: echo "No Reborn Responses API E2E tests selected" exit 1 fi - pytest "${responses_tests[@]}" -v --timeout=120 + scripts/ci/run-hermetic-deterministic-suite.sh command \ + pytest "${responses_tests[@]}" -v --timeout=120 - name: Package live-canary binary id: live_canary_binary @@ -442,7 +454,13 @@ jobs: pip install -e . - name: Run Reborn black-box smoke - run: pytest tests/e2e/scenarios/test_reborn_blackbox_smoke.py -v --timeout=120 + run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command + pytest + tests/e2e/scenarios/test_reborn_blackbox_smoke.py + -v + --timeout=120 reborn-e2e: name: Reborn E2E diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index aae3a94eee3..cda8bb3e57a 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -364,7 +364,8 @@ jobs: # One combined test+lcov invocation, not the two-step `--no-report test` + `report` shape (report has no --workspace/-p, only reports the root package). # shellcheck disable=SC2086 # feature_flags intentionally expands to zero or more Cargo args. - timeout --signal=INT --kill-after=30s 40m \ + scripts/ci/run-hermetic-deterministic-suite.sh command \ + timeout --signal=INT --kill-after=30s 40m \ "${incremental_env[@]}" cargo llvm-cov -p "$package" ${feature_flags} --all-targets \ --lcov --output-path "coverage/${package}.lcov" \ test -- --nocapture @@ -469,7 +470,10 @@ jobs: redis-password: ${{ secrets.SCCACHE_REDIS_PASSWORD }} - name: Run Reborn root tests - run: scripts/ci/run-reborn-root-partition.sh + run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command + scripts/ci/run-reborn-root-partition.sh reborn-group-tests: name: Reborn group tests @@ -561,7 +565,7 @@ jobs: redis-password: ${{ secrets.SCCACHE_REDIS_PASSWORD }} - name: Run Reborn group tests - run: scripts/ci/run-reborn-group-tests.sh + run: scripts/ci/run-hermetic-deterministic-suite.sh groups reborn-integration-coverage: name: Reborn integration coverage (${{ matrix.lane }}) @@ -662,7 +666,8 @@ jobs: # instrumented binaries) keeps the build-cache speed win while # guaranteeing this lane's lcov reflects only its own test run. cargo llvm-cov clean --profraw-only - scripts/ci/reborn-coverage-lane-run.sh part-${{ matrix.lane }}.lcov + scripts/ci/run-hermetic-deterministic-suite.sh command \ + scripts/ci/reborn-coverage-lane-run.sh part-${{ matrix.lane }}.lcov - name: Upload lane lcov artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -814,9 +819,8 @@ jobs: - name: Run Reborn WebUI frontend tests run: | - cd crates/ironclaw_webui/frontend - pnpm install --frozen-lockfile - pnpm test + pnpm --dir crates/ironclaw_webui/frontend install --frozen-lockfile + scripts/ci/run-hermetic-deterministic-suite.sh frontend qa-recorded-fixtures: name: Reborn QA recorded fixtures @@ -866,7 +870,14 @@ jobs: run: scripts/ci/check-reborn-qa-fixtures.sh - name: Run Reborn QA fixture contracts and replay - run: cargo test -p ironclaw_reborn_integration_tests --test reborn_qa_recorded_behavior -- --nocapture + run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command + cargo test + -p ironclaw_reborn_integration_tests + --test reborn_qa_recorded_behavior + -- + --nocapture reborn-tests: name: Tests (Reborn) diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md new file mode 100644 index 00000000000..359e244bfe2 --- /dev/null +++ b/docs/internal/hermetic-deterministic-suite.md @@ -0,0 +1,53 @@ +# Hermetic deterministic suite + +The canonical complete local command for the merge-gating deterministic Reborn +test strategy is: + +```bash +scripts/ci/run-hermetic-deterministic-suite.sh all +``` + +It composes the same checked-in discovery and entrypoints used by +`reborn-tests.yml` and `reborn-e2e.yml`: the production Reborn package closure +and allowlist with CI feature flags, root test partitions, shared-state group +suites, the complete registered Reborn integration tier, recorded-QA fixture +checks and replay, the Rust Reborn E2E gate, WebUI unit tests, standalone binary +build, and the merge-gating Python E2E/provider lanes. It deliberately excludes +live canaries, nightly Playwright shards, stress, release, and platform +compile-only jobs. + +Install the CI-pinned toolchains first. In particular, build the Emulate +revision pinned in `.github/workflows/reborn-e2e.yml`, set +`IRONCLAW_EMULATE_CLI` to its built CLI, install the E2E Python package and +Chromium, install frontend dependencies, and prefetch locked Cargo +dependencies. Cargo and package installation are build-tooling preconditions; +the non-loopback guard is applied to executed test binaries and Python/Node test +processes, not dependency download tooling or remote compiler caches. + +Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. That boundary: + +- removes real provider credentials and ambient provider/LLM behavior; +- gives the process a fresh temporary home, IronClaw base/reborn homes, + workspace, XDG directories, and temporary directory; +- pins timezone, locale, Python hash seed, the test random seed, and the + test-clock epoch; +- suppresses the OS keychain and long provider retries; and +- records and fails on any non-loopback IP connection while permitting Unix + sockets and deliberate IPv4/IPv6 localhost fakes. + +CI may invoke a narrower stage or use `command` to retain its matrix sharding: + +```bash +scripts/ci/run-hermetic-deterministic-suite.sh rust-e2e substrates +scripts/ci/run-hermetic-deterministic-suite.sh command cargo test -p ironclaw_network +``` + +The guard is mutation-tested by: + +```bash +scripts/ci/test-hermetic-test-process.sh +``` + +Maintainers can locally sabotage one control and confirm the self-test turns +red with `IRONCLAW_HERMETIC_SELF_TEST_SABOTAGE` set to `env`, `temp`, +`clock-seed`, or `network`. diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c new file mode 100644 index 00000000000..f30789460d3 --- /dev/null +++ b/scripts/ci/hermetic-network-guard.c @@ -0,0 +1,206 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__APPLE__) +typedef int (*connect_fn)(int, const struct sockaddr *, socklen_t); +typedef ssize_t (*sendmsg_fn)(int, const struct msghdr *, int); +typedef ssize_t (*sendto_fn)(int, const void *, size_t, int, const struct sockaddr *, socklen_t); +static connect_fn real_connect(void) { + static connect_fn function; + if (function == NULL) { + function = (connect_fn)dlsym(RTLD_NEXT, "connect"); + } + return function; +} + +static sendmsg_fn real_sendmsg(void) { + static sendmsg_fn function; + if (function == NULL) { + function = (sendmsg_fn)dlsym(RTLD_NEXT, "sendmsg"); + } + return function; +} + +static sendto_fn real_sendto(void) { + static sendto_fn function; + if (function == NULL) { + function = (sendto_fn)dlsym(RTLD_NEXT, "sendto"); + } + return function; +} +#endif + +static int is_loopback(const struct sockaddr *address) { + if (address == NULL || address->sa_family == AF_UNSPEC) { + return 1; + } + if (address->sa_family == AF_INET) { + const struct sockaddr_in *ipv4 = (const struct sockaddr_in *)address; + return (ntohl(ipv4->sin_addr.s_addr) & 0xff000000U) == 0x7f000000U; + } + if (address->sa_family == AF_INET6) { + const struct sockaddr_in6 *ipv6 = (const struct sockaddr_in6 *)address; + return IN6_IS_ADDR_LOOPBACK(&ipv6->sin6_addr); + } + /* Unix sockets and non-IP kernel transports are local by definition. */ + return 1; +} + +static void record_violation(const struct sockaddr *address) { + const char *path = getenv("IRONCLAW_HERMETIC_NETWORK_VIOLATIONS"); + if (path == NULL || path[0] == '\0') { + return; + } + + char ip[INET6_ADDRSTRLEN] = "unknown"; + if (address != NULL && address->sa_family == AF_INET) { + const struct sockaddr_in *ipv4 = (const struct sockaddr_in *)address; + (void)inet_ntop(AF_INET, &ipv4->sin_addr, ip, sizeof(ip)); + } else if (address != NULL && address->sa_family == AF_INET6) { + const struct sockaddr_in6 *ipv6 = (const struct sockaddr_in6 *)address; + (void)inet_ntop(AF_INET6, &ipv6->sin6_addr, ip, sizeof(ip)); + } + + char message[256]; + int length = snprintf( + message, + sizeof(message), + "non-loopback network attempt: pid=%ld family=%d address=%s\n", + (long)getpid(), + address == NULL ? -1 : address->sa_family, + ip + ); + if (length <= 0) { + return; + } + if ((size_t)length >= sizeof(message)) { + length = (int)sizeof(message) - 1; + } + + int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0600); + if (fd >= 0) { + (void)write(fd, message, (size_t)length); + (void)close(fd); + } +} + +static int guarded_connect( + int socket_fd, + const struct sockaddr *address, + socklen_t address_length +) { + if (!is_loopback(address)) { + record_violation(address); + errno = EPERM; + return -1; + } +#if defined(__APPLE__) + return (int)syscall(SYS_connect, socket_fd, address, address_length); +#else + connect_fn function = real_connect(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(socket_fd, address, address_length); +#endif +} + +static ssize_t guarded_sendmsg(int socket_fd, const struct msghdr *message, int flags) { + const struct sockaddr *address = + message == NULL ? NULL : (const struct sockaddr *)message->msg_name; + if (!is_loopback(address)) { + record_violation(address); + errno = EPERM; + return -1; + } +#if defined(__APPLE__) + return (ssize_t)syscall(SYS_sendmsg, socket_fd, message, flags); +#else + sendmsg_fn function = real_sendmsg(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(socket_fd, message, flags); +#endif +} + +static ssize_t guarded_sendto( + int socket_fd, + const void *buffer, + size_t length, + int flags, + const struct sockaddr *address, + socklen_t address_length +) { + if (!is_loopback(address)) { + record_violation(address); + errno = EPERM; + return -1; + } +#if defined(__APPLE__) + return (ssize_t)syscall( + SYS_sendto, + socket_fd, + buffer, + length, + flags, + address, + address_length + ); +#else + sendto_fn function = real_sendto(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(socket_fd, buffer, length, flags, address, address_length); +#endif +} + +#if defined(__APPLE__) +#define DYLD_INTERPOSE(replacement, replacee) \ + __attribute__((used)) static struct { \ + const void *replacement; \ + const void *replacee; \ + } _interpose_##replacee __attribute__((section("__DATA,__interpose"))) = { \ + (const void *)(unsigned long)&replacement, \ + (const void *)(unsigned long)&replacee \ + } + +DYLD_INTERPOSE(guarded_connect, connect); +DYLD_INTERPOSE(guarded_sendmsg, sendmsg); +DYLD_INTERPOSE(guarded_sendto, sendto); +#else +int connect(int socket_fd, const struct sockaddr *address, socklen_t address_length) { + return guarded_connect(socket_fd, address, address_length); +} + +ssize_t sendmsg(int socket_fd, const struct msghdr *message, int flags) { + return guarded_sendmsg(socket_fd, message, flags); +} + +ssize_t sendto( + int socket_fd, + const void *buffer, + size_t length, + int flags, + const struct sockaddr *address, + socklen_t address_length +) { + return guarded_sendto(socket_fd, buffer, length, flags, address, address_length); +} +#endif diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c new file mode 100644 index 00000000000..6b8d37ed572 --- /dev/null +++ b/scripts/ci/hermetic-network-probe.c @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 2 && argc != 3) { + return 2; + } + + int use_udp = argc == 3 && strcmp(argv[2], "udp") == 0; + int fd = socket(AF_INET, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (fd < 0) { + return 3; + } + int flags = fcntl(fd, F_GETFL, 0); + if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { + close(fd); + return 5; + } + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(9); + if (inet_pton(AF_INET, argv[1], &address.sin_addr) != 1) { + close(fd); + return 4; + } + if (use_udp) { + const char byte = 'x'; + (void)sendto( + fd, + &byte, + sizeof(byte), + 0, + (const struct sockaddr *)&address, + sizeof(address) + ); + } else { + (void)connect(fd, (const struct sockaddr *)&address, sizeof(address)); + } + close(fd); + return 0; +} diff --git a/scripts/ci/hermetic-network-runner.sh b/scripts/ci/hermetic-network-runner.sh new file mode 100755 index 00000000000..432a51cda6d --- /dev/null +++ b/scripts/ci/hermetic-network-runner.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +guard_library="${IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY:?network guard library is not configured}" +hermetic_root="${IRONCLAW_HERMETIC_ROOT:?hermetic root is not configured}" +violation_log="$(mktemp "${hermetic_root}/network-violations.XXXXXX")" +trap 'rm -f "${violation_log}"' EXIT + +export IRONCLAW_HERMETIC_NETWORK_VIOLATIONS="${violation_log}" +case "$(uname -s)" in + Linux) + export LD_PRELOAD="${guard_library}${LD_PRELOAD:+:${LD_PRELOAD}}" + ;; + Darwin) + export DYLD_INSERT_LIBRARIES="${guard_library}${DYLD_INSERT_LIBRARIES:+:${DYLD_INSERT_LIBRARIES}}" + export DYLD_FORCE_FLAT_NAMESPACE=1 + ;; + *) + echo "unsupported platform for hermetic network guard: $(uname -s)" >&2 + exit 2 + ;; +esac + +set +e +"$@" +command_status=$? +set -e + +if [[ -s "${violation_log}" ]]; then + cat "${violation_log}" >&2 + exit 86 +fi +exit "${command_status}" diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh new file mode 100755 index 00000000000..03b4484f62c --- /dev/null +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +hermetic="${repo_root}/scripts/ci/run-hermetic-test-process.sh" +stage="${1:-all}" +if [[ "$#" -gt 0 ]]; then + shift +fi + +run() { + "${hermetic}" -- "$@" +} + +run_root_partitions() { + local partition + for partition in 0 1 2 3; do + REBORN_ROOT_TEST_PARTITIONS=4 \ + REBORN_ROOT_TEST_PARTITION="${partition}" \ + REBORN_ROOT_TEST_TIMEOUT="${REBORN_ROOT_TEST_TIMEOUT:-28m}" \ + RUST_MIN_STACK=67108864 \ + run "${repo_root}/scripts/ci/run-reborn-root-partition.sh" + done +} + +discover_reborn_packages() { + local allowlist closure + allowlist="$( + cargo metadata --no-deps --format-version 1 \ + | jq -c ' + [ + .packages[] + | select( + ( + (.name == "ironclaw") + or (.name == "ironclaw_runner") + or (.name | startswith("ironclaw_reborn")) + or (.name | startswith("ironclaw_product")) + or (.name == "ironclaw_architecture") + or (.name == "ironclaw_slack_extension") + or (.name == "ironclaw_telegram_extension") + or (.name == "ironclaw_telegram_v2_adapter") + or (.name | startswith("ironclaw_webui")) + ) + and (.name != "ironclaw_reborn_integration_tests") + ) + | .name + ] + | unique + ' + )" + closure="$( + comm -12 \ + <(cargo tree -p ironclaw -e normal,build --prefix none \ + | grep -oE 'ironclaw_[a-z0-9_]+' \ + | LC_ALL=C sort -u) \ + <(cargo metadata --no-deps --format-version 1 \ + | jq -r '.packages[].name' \ + | LC_ALL=C sort -u) \ + | jq -R -s -c 'split("\n") | map(select(length > 0))' + )" + jq -n -r \ + --argjson allowlist "${allowlist}" \ + --argjson closure "${closure}" \ + '$allowlist + $closure | unique | .[]' +} + +run_crate_tests() { + local package feature_flags + while IFS= read -r package; do + feature_flags="$("${repo_root}/scripts/ci/package-feature-flags.sh" "${package}")" + # shellcheck disable=SC2086 # feature_flags is the checked-in CI argument list. + run cargo test -p "${package}" ${feature_flags} --all-targets -- --nocapture + done < <(discover_reborn_packages) +} + +run_integration_tier() { + local test_name + while IFS= read -r test_name; do + [[ "${test_name}" == --test ]] && continue + run cargo test -p ironclaw_reborn_integration_tests \ + --test "${test_name}" -- --nocapture + done < <("${repo_root}/scripts/ci/reborn-coverage-int-tier-tests.sh") +} + +run_python_e2e() { + if [[ -z "${IRONCLAW_EMULATE_CLI:-}" ]]; then + echo "IRONCLAW_EMULATE_CLI must name the built pinned Emulate CLI" >&2 + echo "Use the revision pinned in .github/workflows/reborn-e2e.yml." >&2 + return 2 + fi + run pytest \ + tests/e2e/scenarios/test_reborn_webui_v2_smoke.py \ + -v --timeout=120 + run pytest \ + tests/e2e/scenarios/test_provider_capability_inventory.py \ + tests/e2e/scenarios/test_journey_coverage.py \ + tests/e2e/scenarios/test_emulate_reborn_provider_contracts.py \ + tests/e2e/scenarios/test_provider_fault_proxy.py \ + tests/e2e/scenarios/test_emulate_build_parity.py \ + tests/e2e/scenarios/test_provider_world_isolation.py \ + tests/e2e/scenarios/test_reborn_qa_trace_replay.py \ + tests/e2e/scenarios/test_reborn_qa_trace_full_path.py \ + -m "not shared_world" \ + -v --timeout=120 + run pytest tests/e2e/scenarios/test_reborn_blackbox_smoke.py -v --timeout=120 +} + +case "${stage}" in + self-test) + "${repo_root}/scripts/ci/test-hermetic-test-process.sh" + ;; + root) + run_root_partitions + ;; + crates) + run_crate_tests + ;; + groups) + REBORN_GROUP_TEST_TIMEOUT="${REBORN_GROUP_TEST_TIMEOUT:-28m}" \ + RUST_MIN_STACK=67108864 \ + run "${repo_root}/scripts/ci/run-reborn-group-tests.sh" + ;; + integration) + run_integration_tier + ;; + qa) + run "${repo_root}/scripts/ci/check-reborn-qa-fixtures.sh" + run cargo test -p ironclaw_reborn_integration_tests \ + --test reborn_qa_recorded_behavior -- --nocapture + ;; + rust-e2e) + run "${repo_root}/scripts/reborn-e2e-rust.sh" "${1:-all}" + ;; + frontend) + run pnpm --dir crates/ironclaw_webui/frontend test + ;; + python-e2e) + run_python_e2e + ;; + all) + "${repo_root}/scripts/ci/test-hermetic-test-process.sh" + run_crate_tests + run_root_partitions + REBORN_GROUP_TEST_TIMEOUT="${REBORN_GROUP_TEST_TIMEOUT:-28m}" \ + RUST_MIN_STACK=67108864 \ + run "${repo_root}/scripts/ci/run-reborn-group-tests.sh" + run_integration_tier + run "${repo_root}/scripts/ci/check-reborn-qa-fixtures.sh" + run cargo test -p ironclaw_reborn_integration_tests \ + --test reborn_qa_recorded_behavior -- --nocapture + run "${repo_root}/scripts/reborn-e2e-rust.sh" all + run pnpm --dir crates/ironclaw_webui/frontend test + run cargo build -p ironclaw --bin ironclaw + run_python_e2e + ;; + command) + if [[ "$#" -eq 0 ]]; then + echo "command stage requires a command" >&2 + exit 2 + fi + run "$@" + ;; + *) + echo "unknown hermetic deterministic-suite stage: ${stage}" >&2 + echo "expected: all, self-test, crates, root, groups, integration, qa, rust-e2e, frontend, python-e2e, command" >&2 + exit 2 + ;; +esac diff --git a/scripts/ci/run-hermetic-test-process.sh b/scripts/ci/run-hermetic-test-process.sh new file mode 100755 index 00000000000..5ad16d12d0c --- /dev/null +++ b/scripts/ci/run-hermetic-test-process.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${1:-}" != "--" || "$#" -lt 2 ]]; then + echo "usage: scripts/ci/run-hermetic-test-process.sh -- COMMAND [ARGS...]" >&2 + exit 2 +fi +shift + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +network_runner="${repo_root}/scripts/ci/hermetic-network-runner.sh" +network_source="${repo_root}/scripts/ci/hermetic-network-guard.c" +sabotage="${IRONCLAW_HERMETIC_SABOTAGE:-}" + +temp_parent="${RUNNER_TEMP:-${TMPDIR:-/tmp}}" +hermetic_root="$(mktemp -d "${temp_parent%/}/ironclaw-hermetic.XXXXXX")" +trap 'rm -rf "${hermetic_root}"' EXIT + +mkdir -p \ + "${hermetic_root}/home" \ + "${hermetic_root}/base" \ + "${hermetic_root}/reborn-home" \ + "${hermetic_root}/workspace" \ + "${hermetic_root}/tmp" \ + "${hermetic_root}/xdg-cache" \ + "${hermetic_root}/xdg-config" \ + "${hermetic_root}/xdg-data" + +guard_library="" +if [[ "${sabotage}" != "network" ]]; then + case "$(uname -s)" in + Linux) + guard_library="${hermetic_root}/hermetic-network-guard.so" + "${CC:-cc}" -shared -fPIC -O2 -Wall -Wextra -Werror \ + -o "${guard_library}" "${network_source}" -ldl + ;; + Darwin) + guard_library="${hermetic_root}/hermetic-network-guard.dylib" + "${CC:-cc}" -dynamiclib -fPIC -O2 -Wall -Wextra -Werror \ + -Wno-deprecated-declarations \ + -o "${guard_library}" "${network_source}" + ;; + *) + echo "unsupported platform for hermetic network guard: $(uname -s)" >&2 + exit 2 + ;; + esac +fi + +env_args=() +if [[ "${sabotage}" != "env" ]]; then + while IFS='=' read -r key _; do + case "${key}" in + IRONCLAW_EMULATE_CLI|IRONCLAW_GENERATED_SEQUENCE_DEPTH|IRONCLAW_JOURNEY_ORDER) + ;; + IRONCLAW_*) + env_args+=("-u" "${key}") + ;; + *_API_KEY|*_TOKEN|*_SECRET|*_CREDENTIALS|*_PASSWORD|*_PRIVATE_KEY) + env_args+=("-u" "${key}") + ;; + ANTHROPIC_*|OPENAI_*|NEARAI_*|GOOGLE_*|GITHUB_TOKEN|GH_TOKEN|SLACK_*|TELEGRAM_*|NOTION_*|EXA_*|BRAVE_*|AWS_*|AZURE_*|COHERE_*|MISTRAL_*|GROQ_*|\ + LLM_*|OLLAMA_*|REBORN_TOOL_DISCLOSURE|DATABASE_URL|LIBSQL_PATH|SECRETS_MASTER_KEY|\ + HTTP_PROXY|HTTPS_PROXY|ALL_PROXY|NO_PROXY|http_proxy|https_proxy|all_proxy|no_proxy) + env_args+=("-u" "${key}") + ;; + esac + done < <(env) +fi + +original_home="${HOME:-}" +if [[ -n "${CARGO_HOME:-}" ]]; then + cargo_home="${CARGO_HOME}" +elif [[ -n "${original_home}" ]]; then + cargo_home="${original_home}/.cargo" +else + cargo_home="" +fi +if [[ -n "${RUSTUP_HOME:-}" ]]; then + rustup_home="${RUSTUP_HOME}" +elif [[ -n "${original_home}" ]]; then + rustup_home="${original_home}/.rustup" +else + rustup_home="" +fi + +env_args+=( + "IRONCLAW_HERMETIC_ROOT=${hermetic_root}" + "IRONCLAW_DISABLE_OS_KEYCHAIN=1" + "LLM_MAX_RETRIES=0" + "IRONCLAW_REBORN_MODEL_AVAILABILITY_RETRY_ATTEMPTS=1" + "NO_PROXY=127.0.0.1,localhost,::1" + "no_proxy=127.0.0.1,localhost,::1" + "TZ=UTC" + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" +) + +if [[ "${sabotage}" != "temp" ]]; then + env_args+=( + "HOME=${hermetic_root}/home" + "IRONCLAW_BASE_DIR=${hermetic_root}/base" + "IRONCLAW_REBORN_HOME=${hermetic_root}/reborn-home" + "IRONCLAW_TEST_WORKSPACE=${hermetic_root}/workspace" + "TMPDIR=${hermetic_root}/tmp" + "XDG_CACHE_HOME=${hermetic_root}/xdg-cache" + "XDG_CONFIG_HOME=${hermetic_root}/xdg-config" + "XDG_DATA_HOME=${hermetic_root}/xdg-data" + ) +fi +if [[ -n "${cargo_home}" ]]; then + env_args+=("CARGO_HOME=${cargo_home}") +fi +if [[ -n "${rustup_home}" ]]; then + env_args+=("RUSTUP_HOME=${rustup_home}") +fi + +if [[ "${sabotage}" != "clock-seed" ]]; then + env_args+=( + "PYTHONHASHSEED=0" + "IRONCLAW_TEST_RANDOM_SEED=6524" + "IRONCLAW_TEST_CLOCK=2026-01-01T00:00:00Z" + "SOURCE_DATE_EPOCH=1767225600" + ) +fi + +if [[ "${sabotage}" != "network" ]]; then + host_triple="$(rustc -vV | sed -n 's/^host: //p')" + if [[ -z "${host_triple}" ]]; then + echo "unable to determine Rust host triple for the hermetic test runner" >&2 + exit 1 + fi + cargo_runner_key="CARGO_TARGET_$(tr '[:lower:]-' '[:upper:]_' <<<"${host_triple}")_RUNNER" + env_args+=( + "IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY=${guard_library}" + "${cargo_runner_key}=${network_runner}" + ) +fi + +command_prefix=() +command_name="$(basename "$1")" +case "${command_name}" in + python|python[0-9]*|pytest|node|nodejs|npm|npx|pnpm|hermetic-network-probe) + if [[ "${sabotage}" != "network" ]]; then + command_prefix=("${network_runner}") + fi + ;; +esac + +set +e +if [[ "${#command_prefix[@]}" -gt 0 ]]; then + env "${env_args[@]}" "${command_prefix[@]}" "$@" +else + env "${env_args[@]}" "$@" +fi +command_status=$? +set -e + +exit "${command_status}" diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh new file mode 100755 index 00000000000..431811ed0e3 --- /dev/null +++ b/scripts/ci/test-hermetic-test-process.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +runner="${repo_root}/scripts/ci/run-hermetic-test-process.sh" +sabotage="${IRONCLAW_HERMETIC_SELF_TEST_SABOTAGE:-}" +probe_dir="$(mktemp -d "${TMPDIR:-/tmp}/hermetic-network-probe.XXXXXX")" +trap 'rm -rf "${probe_dir}"' EXIT +network_probe="${probe_dir}/hermetic-network-probe" +"${CC:-cc}" -O2 -Wall -Wextra -Werror \ + -o "${network_probe}" "${repo_root}/scripts/ci/hermetic-network-probe.c" + +if [[ ! -x "${runner}" ]]; then + echo "hermetic test-process runner is missing or not executable: ${runner}" >&2 + exit 1 +fi + +run_probe() { + env \ + ANTHROPIC_API_KEY="must-not-leak" \ + GITHUB_TOKEN="must-not-leak" \ + GOOGLE_APPLICATION_CREDENTIALS="/developer/credential.json" \ + LLM_BACKEND="ambient-provider" \ + REBORN_TOOL_DISCLOSURE="Bridged" \ + IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ + "${runner}" -- bash -c ' + set -euo pipefail + for key in \ + ANTHROPIC_API_KEY \ + GITHUB_TOKEN \ + GOOGLE_APPLICATION_CREDENTIALS \ + LLM_BACKEND \ + REBORN_TOOL_DISCLOSURE + do + if [[ -n "${!key+x}" ]]; then + echo "ambient variable leaked into hermetic process: ${key}" >&2 + exit 31 + fi + done + + case "${HOME}" in + "${IRONCLAW_HERMETIC_ROOT}"/*) ;; + *) + echo "HOME is outside the hermetic root: ${HOME}" >&2 + exit 32 + ;; + esac + for path in \ + "${IRONCLAW_BASE_DIR}" \ + "${IRONCLAW_REBORN_HOME}" \ + "${IRONCLAW_TEST_WORKSPACE}" \ + "${TMPDIR}" + do + case "${path}" in + "${IRONCLAW_HERMETIC_ROOT}"/*) ;; + *) + echo "mutable path is outside the hermetic root: ${path}" >&2 + exit 33 + ;; + esac + done + + [[ "${TZ}" == "UTC" ]] + [[ "${LANG}" == "C.UTF-8" ]] + [[ "${LC_ALL}" == "C.UTF-8" ]] + if [[ "${PYTHONHASHSEED:-}" != "0" ]]; then + echo "deterministic Python hash seed is not injected" >&2 + exit 34 + fi + if [[ "${IRONCLAW_TEST_RANDOM_SEED:-}" != "6524" ]]; then + echo "deterministic IronClaw random seed is not injected" >&2 + exit 35 + fi + if [[ "${IRONCLAW_TEST_CLOCK:-}" != "2026-01-01T00:00:00Z" ]]; then + echo "controllable IronClaw test clock is not injected" >&2 + exit 36 + fi + if [[ "${SOURCE_DATE_EPOCH:-}" != "1767225600" ]]; then + echo "deterministic source epoch is not injected" >&2 + exit 37 + fi + + printf "%s\n" "${IRONCLAW_HERMETIC_ROOT}" + ' +} + +first_root="$(run_probe)" +second_root="$(run_probe)" +if [[ "${first_root}" == "${second_root}" ]]; then + echo "hermetic invocations reused mutable state root: ${first_root}" >&2 + exit 1 +fi + +parallel_dir="${probe_dir}/parallel-roots" +mkdir -p "${parallel_dir}" +parallel_pids=() +for index in 1 2 3 4; do + run_probe > "${parallel_dir}/${index}" & + parallel_pids+=("$!") +done +for pid in "${parallel_pids[@]}"; do + wait "${pid}" +done +parallel_root_count="$(LC_ALL=C sort -u "${parallel_dir}"/* | wc -l | tr -d '[:space:]')" +if [[ "${parallel_root_count}" != "4" ]]; then + echo "parallel hermetic invocations did not receive four isolated roots" >&2 + LC_ALL=C sort "${parallel_dir}"/* >&2 + exit 1 +fi + +set +e +network_output="$( + IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ + "${runner}" -- "${network_probe}" 192.0.2.1 2>&1 +)" +network_status=$? +set -e +if [[ "${network_status}" -eq 0 ]]; then + echo "unexpected non-loopback network attempt was not reported" >&2 + exit 1 +fi +if [[ "${network_output}" != *"non-loopback network attempt"* ]]; then + echo "network guard failed without its actionable diagnostic" >&2 + printf '%s\n' "${network_output}" >&2 + exit 1 +fi + +set +e +udp_output="$( + IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ + "${runner}" -- "${network_probe}" 192.0.2.1 udp 2>&1 +)" +udp_status=$? +set -e +if [[ "${udp_status}" -eq 0 || "${udp_output}" != *"non-loopback network attempt"* ]]; then + echo "unexpected non-loopback UDP attempt was not reported" >&2 + printf '%s\n' "${udp_output}" >&2 + exit 1 +fi + +# Deliberate localhost fakes remain usable. A refused port is sufficient: the +# guard must allow the connect syscall to reach the kernel rather than report it. +"${runner}" -- "${network_probe}" 127.0.0.1 + +"${runner}" -- python3 - "${repo_root}/tests/e2e" <<'PY' +import sys + +sys.path.insert(0, sys.argv[1]) +from hermetic_process import forward_hermetic_process_env + +source = { + "IRONCLAW_HERMETIC_NETWORK_VIOLATIONS": "/tmp/violations", + "LD_PRELOAD": "/tmp/guard.so", + "ANTHROPIC_API_KEY": "must-not-forward", +} +child = {} +forward_hermetic_process_env(child, source) +assert child == { + "IRONCLAW_HERMETIC_NETWORK_VIOLATIONS": "/tmp/violations", + "LD_PRELOAD": "/tmp/guard.so", +} +PY + +set +e +child_output="$( + "${runner}" -- python3 - "${repo_root}/tests/e2e" "${network_probe}" 2>&1 <<'PY' +import subprocess +import sys + +sys.path.insert(0, sys.argv[1]) +from hermetic_process import forward_hermetic_process_env + +child = {} +forward_hermetic_process_env(child) +subprocess.run([sys.argv[2], "192.0.2.1"], env=child, check=True) +PY +)" +child_status=$? +set -e +if [[ "${child_status}" -eq 0 || "${child_output}" != *"non-loopback network attempt"* ]]; then + echo "minimal-env E2E child lost the non-loopback network guard" >&2 + printf '%s\n' "${child_output}" >&2 + exit 1 +fi + +for workflow_contract in \ + ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh groups" \ + ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh command" \ + ".github/workflows/reborn-e2e.yml:scripts/ci/run-hermetic-deterministic-suite.sh" \ + ".github/workflows/code_style.yml:scripts/ci/test-hermetic-test-process.sh" +do + workflow="${workflow_contract%%:*}" + needle="${workflow_contract#*:}" + if ! rg -Fq "${needle}" "${repo_root}/${workflow}"; then + echo "CI/local hermetic-suite parity lost: ${workflow} lacks '${needle}'" >&2 + exit 1 + fi +done + +for stage_call in \ + "run_crate_tests" \ + "run_root_partitions" \ + "run_integration_tier" \ + "run-reborn-group-tests.sh" \ + "check-reborn-qa-fixtures.sh" \ + "reborn-e2e-rust.sh" \ + "run_python_e2e" +do + if ! rg -Fq "${stage_call}" "${repo_root}/scripts/ci/run-hermetic-deterministic-suite.sh"; then + echo "canonical complete suite lost required stage: ${stage_call}" >&2 + exit 1 + fi +done + +echo "hermetic test-process self-test: OK" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ae25cc3dac4..c5cc2728d7c 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -21,6 +21,7 @@ import httpx import pytest +from hermetic_process import forward_hermetic_process_env from helpers import ( AUTH_TOKEN, EMULATE_GITHUB_BEARER, @@ -316,12 +317,13 @@ def test_tool_zips() -> dict[str, Path]: def _forward_coverage_env(env: dict[str, str]) -> None: - """Forward cargo-llvm-cov env vars into child processes when present.""" + """Forward CI instrumentation and hermetic controls to child processes.""" cov_env_prefixes = ("CARGO_LLVM_COV", "LLVM_") cov_env_extras = ("CARGO_ENCODED_RUSTFLAGS", "CARGO_INCREMENTAL") for key, val in os.environ.items(): if key.startswith(cov_env_prefixes) or key in cov_env_extras: env[key] = val + forward_hermetic_process_env(env) def _build_gateway_env( diff --git a/tests/e2e/hermetic_process.py b/tests/e2e/hermetic_process.py new file mode 100644 index 00000000000..e7a3feeb7c1 --- /dev/null +++ b/tests/e2e/hermetic_process.py @@ -0,0 +1,36 @@ +"""Process-boundary controls shared by the deterministic E2E harnesses.""" + +from __future__ import annotations + +import os +import sys +from collections.abc import Mapping, MutableMapping + +_HERMETIC_PROCESS_ENV_KEYS = ( + "IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY", + "IRONCLAW_HERMETIC_NETWORK_VIOLATIONS", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + "DYLD_FORCE_FLAT_NAMESPACE", +) + + +def forward_hermetic_process_env( + env: MutableMapping[str, str], + source: Mapping[str, str] | None = None, +) -> None: + """Forward only the syscall guard controls into a minimal child env.""" + source_env = os.environ if source is None else source + for key in _HERMETIC_PROCESS_ENV_KEYS: + value = source_env.get(key) + if value is not None: + env[key] = value + + guard_library = source_env.get("IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY") + if guard_library is None: + return + if sys.platform == "darwin": + env.setdefault("DYLD_INSERT_LIBRARIES", guard_library) + env.setdefault("DYLD_FORCE_FLAT_NAMESPACE", "1") + elif sys.platform.startswith("linux"): + env.setdefault("LD_PRELOAD", guard_library) diff --git a/tests/e2e/reborn_webui_harness.py b/tests/e2e/reborn_webui_harness.py index dc6d091d922..ef3db02bb1d 100644 --- a/tests/e2e/reborn_webui_harness.py +++ b/tests/e2e/reborn_webui_harness.py @@ -22,6 +22,7 @@ from playwright.async_api import Error as PlaywrightError from fixtures.mock_oauth_idp import MockOidcProfile, start_mock_oauth_idp +from hermetic_process import forward_hermetic_process_env from helpers import REBORN_V2_AUTH_TOKEN, SEL_V2, wait_for_ready USER_ID = "reborn-v2-e2e-user" @@ -380,6 +381,7 @@ def forward_coverage_env(env: dict[str, str]) -> None: "CARGO_INCREMENTAL", }: env[key] = value + forward_hermetic_process_env(env) async def stop_process(proc, *, sig=signal.SIGINT, timeout: float = 10) -> None: diff --git a/tests/e2e/scenarios/test_reborn_responses_api.py b/tests/e2e/scenarios/test_reborn_responses_api.py index a592641a040..01d00874754 100644 --- a/tests/e2e/scenarios/test_reborn_responses_api.py +++ b/tests/e2e/scenarios/test_reborn_responses_api.py @@ -11,6 +11,7 @@ import httpx import pytest +from hermetic_process import forward_hermetic_process_env from helpers import REBORN_V2_AUTH_TOKEN, wait_for_ready USER_ID = "reborn-responses-e2e-user" @@ -37,6 +38,7 @@ def _forward_coverage_env(env: dict[str, str]) -> None: "CARGO_INCREMENTAL", }: env[key] = value + forward_hermetic_process_env(env) async def _stop_process(proc, *, sig=signal.SIGINT, timeout: float = 10) -> None: From d7a04c4d4a4b8bdb8df48a5aad8652194691d219 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 11:16:50 +0300 Subject: [PATCH 02/22] fix(ci): close hermetic process guard gaps --- docs/internal/hermetic-deterministic-suite.md | 25 ++++++++++++------ scripts/ci/hermetic-network-guard.c | 20 ++++++++++++-- scripts/ci/hermetic-network-probe.c | 15 +++++++++-- scripts/ci/hermetic-network-runner.sh | 24 ++++++++++++++--- .../ci/run-hermetic-deterministic-suite.sh | 14 ++++++++++ scripts/ci/run-hermetic-test-process.sh | 22 +++++++--------- scripts/ci/test-hermetic-test-process.sh | 26 ++++++++++--------- 7 files changed, 106 insertions(+), 40 deletions(-) diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md index 359e244bfe2..9a5f49e9880 100644 --- a/docs/internal/hermetic-deterministic-suite.md +++ b/docs/internal/hermetic-deterministic-suite.md @@ -20,20 +20,29 @@ Install the CI-pinned toolchains first. In particular, build the Emulate revision pinned in `.github/workflows/reborn-e2e.yml`, set `IRONCLAW_EMULATE_CLI` to its built CLI, install the E2E Python package and Chromium, install frontend dependencies, and prefetch locked Cargo -dependencies. Cargo and package installation are build-tooling preconditions; -the non-loopback guard is applied to executed test binaries and Python/Node test -processes, not dependency download tooling or remote compiler caches. +dependencies. Package installation is a build-tooling precondition. The +canonical script performs `cargo fetch --locked` before entering the guarded +process and then forces Cargo offline. Every command and descendant inside the +suite is guarded, so remote compiler wrappers are disabled rather than treated +as a network exception. Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. That boundary: - removes real provider credentials and ambient provider/LLM behavior; - gives the process a fresh temporary home, IronClaw base/reborn homes, workspace, XDG directories, and temporary directory; -- pins timezone, locale, Python hash seed, the test random seed, and the - test-clock epoch; +- pins timezone, locale, and Python hash iteration order; - suppresses the OS keychain and long provider retries; and -- records and fails on any non-loopback IP connection while permitting Unix - sockets and deliberate IPv4/IPv6 localhost fakes. +- denies non-loopback IP connections while permitting Unix sockets and + deliberate IPv4/IPv6 localhost fakes. The syscall interposer records denied + attempts on Linux and ordinary macOS binaries; macOS additionally uses the + process sandbox so SIP-protected launchers remain fail-closed. + +Rust wall-clock and random behavior is not overridden through process-global +environment variables. Time-sensitive domain tests use their owning typed +clock seams (for example `FakeClock`/`FixedClock`), configured Reborn jitter +defaults to zero in deterministic tests, and cryptographic or identity +randomness remains OS-backed. CI may invoke a narrower stage or use `command` to retain its matrix sharding: @@ -50,4 +59,4 @@ scripts/ci/test-hermetic-test-process.sh Maintainers can locally sabotage one control and confirm the self-test turns red with `IRONCLAW_HERMETIC_SELF_TEST_SABOTAGE` set to `env`, `temp`, -`clock-seed`, or `network`. +`python-seed`, or `network`. diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c index f30789460d3..eb74a984f03 100644 --- a/scripts/ci/hermetic-network-guard.c +++ b/scripts/ci/hermetic-network-guard.c @@ -58,6 +58,21 @@ static int is_loopback(const struct sockaddr *address) { return 1; } +static void write_violation(int fd, const char *message, size_t length) { + size_t offset = 0; + while (offset < length) { + ssize_t written = write(fd, message + offset, length - offset); + if (written > 0) { + offset += (size_t)written; + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + break; + } +} + static void record_violation(const struct sockaddr *address) { const char *path = getenv("IRONCLAW_HERMETIC_NETWORK_VIOLATIONS"); if (path == NULL || path[0] == '\0') { @@ -91,8 +106,9 @@ static void record_violation(const struct sockaddr *address) { int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0600); if (fd >= 0) { - (void)write(fd, message, (size_t)length); - (void)close(fd); + write_violation(fd, message, (size_t)length); + int close_status = close(fd); + (void)close_status; } } diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c index 6b8d37ed572..82e8a10c302 100644 --- a/scripts/ci/hermetic-network-probe.c +++ b/scripts/ci/hermetic-network-probe.c @@ -1,6 +1,8 @@ #include +#include #include #include +#include #include #include #include @@ -28,9 +30,10 @@ int main(int argc, char **argv) { close(fd); return 4; } + int network_status; if (use_udp) { const char byte = 'x'; - (void)sendto( + network_status = (int)sendto( fd, &byte, sizeof(byte), @@ -39,7 +42,15 @@ int main(int argc, char **argv) { sizeof(address) ); } else { - (void)connect(fd, (const struct sockaddr *)&address, sizeof(address)); + network_status = connect(fd, (const struct sockaddr *)&address, sizeof(address)); + } + if (network_status < 0 && errno == EPERM) { + fprintf( + stderr, + "non-loopback network attempt rejected by hermetic process boundary\n" + ); + close(fd); + return 90; } close(fd); return 0; diff --git a/scripts/ci/hermetic-network-runner.sh b/scripts/ci/hermetic-network-runner.sh index 432a51cda6d..381b12c4fb8 100755 --- a/scripts/ci/hermetic-network-runner.sh +++ b/scripts/ci/hermetic-network-runner.sh @@ -5,15 +5,33 @@ guard_library="${IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY:?network guard library hermetic_root="${IRONCLAW_HERMETIC_ROOT:?hermetic root is not configured}" violation_log="$(mktemp "${hermetic_root}/network-violations.XXXXXX")" trap 'rm -f "${violation_log}"' EXIT +guarded_command=("$@") export IRONCLAW_HERMETIC_NETWORK_VIOLATIONS="${violation_log}" case "$(uname -s)" in Linux) - export LD_PRELOAD="${guard_library}${LD_PRELOAD:+:${LD_PRELOAD}}" + case ":${LD_PRELOAD:-}:" in + *":${guard_library}:"*) ;; + *) export LD_PRELOAD="${guard_library}${LD_PRELOAD:+:${LD_PRELOAD}}" ;; + esac ;; Darwin) - export DYLD_INSERT_LIBRARIES="${guard_library}${DYLD_INSERT_LIBRARIES:+:${DYLD_INSERT_LIBRARIES}}" + case ":${DYLD_INSERT_LIBRARIES:-}:" in + *":${guard_library}:"*) ;; + *) + export DYLD_INSERT_LIBRARIES="${guard_library}${DYLD_INSERT_LIBRARIES:+:${DYLD_INSERT_LIBRARIES}}" + ;; + esac export DYLD_FORCE_FLAT_NAMESPACE=1 + # SIP-protected Apple executables strip DYLD_* before their descendants + # inherit it. The process sandbox keeps the whole tree fail-closed; the + # interposer still records actionable diagnostics for ordinary binaries. + guarded_command=( + sandbox-exec + -p + '(version 1) (allow default) (deny network-outbound) (allow network-outbound (remote ip "localhost:*"))' + "$@" + ) ;; *) echo "unsupported platform for hermetic network guard: $(uname -s)" >&2 @@ -22,7 +40,7 @@ case "$(uname -s)" in esac set +e -"$@" +"${guarded_command[@]}" command_status=$? set -e diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index 03b4484f62c..114aa03e19f 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -12,6 +12,12 @@ run() { "${hermetic}" -- "$@" } +prepare_rust_dependencies() { + # Dependency acquisition is setup, not test behavior. Fetch once before the + # hermetic process switches Cargo into offline mode. + cargo fetch --locked +} + run_root_partitions() { local partition for partition in 0 1 2 3; do @@ -111,25 +117,31 @@ case "${stage}" in "${repo_root}/scripts/ci/test-hermetic-test-process.sh" ;; root) + prepare_rust_dependencies run_root_partitions ;; crates) + prepare_rust_dependencies run_crate_tests ;; groups) + prepare_rust_dependencies REBORN_GROUP_TEST_TIMEOUT="${REBORN_GROUP_TEST_TIMEOUT:-28m}" \ RUST_MIN_STACK=67108864 \ run "${repo_root}/scripts/ci/run-reborn-group-tests.sh" ;; integration) + prepare_rust_dependencies run_integration_tier ;; qa) + prepare_rust_dependencies run "${repo_root}/scripts/ci/check-reborn-qa-fixtures.sh" run cargo test -p ironclaw_reborn_integration_tests \ --test reborn_qa_recorded_behavior -- --nocapture ;; rust-e2e) + prepare_rust_dependencies run "${repo_root}/scripts/reborn-e2e-rust.sh" "${1:-all}" ;; frontend) @@ -140,6 +152,7 @@ case "${stage}" in ;; all) "${repo_root}/scripts/ci/test-hermetic-test-process.sh" + prepare_rust_dependencies run_crate_tests run_root_partitions REBORN_GROUP_TEST_TIMEOUT="${REBORN_GROUP_TEST_TIMEOUT:-28m}" \ @@ -159,6 +172,7 @@ case "${stage}" in echo "command stage requires a command" >&2 exit 2 fi + prepare_rust_dependencies run "$@" ;; *) diff --git a/scripts/ci/run-hermetic-test-process.sh b/scripts/ci/run-hermetic-test-process.sh index 5ad16d12d0c..2222f1fb69b 100755 --- a/scripts/ci/run-hermetic-test-process.sh +++ b/scripts/ci/run-hermetic-test-process.sh @@ -65,7 +65,7 @@ if [[ "${sabotage}" != "env" ]]; then env_args+=("-u" "${key}") ;; esac - done < <(env) + done <<<"$(env)" fi original_home="${HOME:-}" @@ -91,6 +91,7 @@ env_args+=( "IRONCLAW_REBORN_MODEL_AVAILABILITY_RETRY_ATTEMPTS=1" "NO_PROXY=127.0.0.1,localhost,::1" "no_proxy=127.0.0.1,localhost,::1" + "CARGO_NET_OFFLINE=true" "TZ=UTC" "LANG=C.UTF-8" "LC_ALL=C.UTF-8" @@ -115,12 +116,9 @@ if [[ -n "${rustup_home}" ]]; then env_args+=("RUSTUP_HOME=${rustup_home}") fi -if [[ "${sabotage}" != "clock-seed" ]]; then +if [[ "${sabotage}" != "python-seed" ]]; then env_args+=( "PYTHONHASHSEED=0" - "IRONCLAW_TEST_RANDOM_SEED=6524" - "IRONCLAW_TEST_CLOCK=2026-01-01T00:00:00Z" - "SOURCE_DATE_EPOCH=1767225600" ) fi @@ -138,14 +136,12 @@ if [[ "${sabotage}" != "network" ]]; then fi command_prefix=() -command_name="$(basename "$1")" -case "${command_name}" in - python|python[0-9]*|pytest|node|nodejs|npm|npx|pnpm|hermetic-network-probe) - if [[ "${sabotage}" != "network" ]]; then - command_prefix=("${network_runner}") - fi - ;; -esac +if [[ "${sabotage}" != "network" ]]; then + command_prefix=("${network_runner}") + # Fetch dependencies and prepare compiler caches before this boundary. + # A remote compiler wrapper must not become a hidden network exception. + env_args+=("RUSTC_WRAPPER=") +fi set +e if [[ "${#command_prefix[@]}" -gt 0 ]]; then diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 431811ed0e3..bd6ceb3dd02 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -67,18 +67,6 @@ run_probe() { echo "deterministic Python hash seed is not injected" >&2 exit 34 fi - if [[ "${IRONCLAW_TEST_RANDOM_SEED:-}" != "6524" ]]; then - echo "deterministic IronClaw random seed is not injected" >&2 - exit 35 - fi - if [[ "${IRONCLAW_TEST_CLOCK:-}" != "2026-01-01T00:00:00Z" ]]; then - echo "controllable IronClaw test clock is not injected" >&2 - exit 36 - fi - if [[ "${SOURCE_DATE_EPOCH:-}" != "1767225600" ]]; then - echo "deterministic source epoch is not injected" >&2 - exit 37 - fi printf "%s\n" "${IRONCLAW_HERMETIC_ROOT}" ' @@ -119,6 +107,20 @@ if [[ "${network_status}" -eq 0 ]]; then echo "unexpected non-loopback network attempt was not reported" >&2 exit 1 fi + +# The guard applies to arbitrary launchers, not just known test executables. +set +e +shell_output="$( + IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ + "${runner}" -- bash -c '"$1" 192.0.2.1' _ "${network_probe}" 2>&1 +)" +shell_status=$? +set -e +if [[ "${shell_status}" -eq 0 || "${shell_output}" != *"non-loopback network attempt"* ]]; then + echo "shell-launched non-loopback network attempt was not reported" >&2 + printf '%s\n' "${shell_output}" >&2 + exit 1 +fi if [[ "${network_output}" != *"non-loopback network attempt"* ]]; then echo "network guard failed without its actionable diagnostic" >&2 printf '%s\n' "${network_output}" >&2 From ee24c496d9a55d5e54b4099974437f37954cd6a7 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 11:19:14 +0300 Subject: [PATCH 03/22] fix(ci): preserve frontend package-manager selection --- .github/workflows/reborn-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index cda8bb3e57a..845f75d07a0 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -819,7 +819,10 @@ jobs: - name: Run Reborn WebUI frontend tests run: | - pnpm --dir crates/ironclaw_webui/frontend install --frozen-lockfile + ( + cd crates/ironclaw_webui/frontend + pnpm install --frozen-lockfile + ) scripts/ci/run-hermetic-deterministic-suite.sh frontend qa-recorded-fixtures: From f7118956ed4ae4eab7595c95f8d982fb3e580018 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 11:25:02 +0300 Subject: [PATCH 04/22] fix(ci): prepare frontend dependencies outside guard --- .github/workflows/reborn-tests.yml | 7 +-- .../ci/run-hermetic-deterministic-suite.sh | 46 ++++++++++++++++++- scripts/ci/test-hermetic-test-process.sh | 4 +- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index 845f75d07a0..f5ffcf09dd2 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -818,12 +818,7 @@ jobs: run: corepack enable pnpm - name: Run Reborn WebUI frontend tests - run: | - ( - cd crates/ironclaw_webui/frontend - pnpm install --frozen-lockfile - ) - scripts/ci/run-hermetic-deterministic-suite.sh frontend + run: scripts/ci/run-hermetic-deterministic-suite.sh frontend qa-recorded-fixtures: name: Reborn QA recorded fixtures diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index 114aa03e19f..9db4cac6074 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -8,6 +8,15 @@ if [[ "$#" -gt 0 ]]; then shift fi +frontend_corepack_home="" + +cleanup() { + if [[ -n "${frontend_corepack_home}" && -d "${frontend_corepack_home}" ]]; then + rm -rf -- "${frontend_corepack_home}" + fi +} +trap cleanup EXIT + run() { "${hermetic}" -- "$@" } @@ -112,6 +121,37 @@ run_python_e2e() { run pytest tests/e2e/scenarios/test_reborn_blackbox_smoke.py -v --timeout=120 } +prepare_frontend_dependencies() { + local package_manager + package_manager="$( + jq -r '.packageManager' \ + "${repo_root}/crates/ironclaw_webui/frontend/package.json" + )" + if [[ "${package_manager}" != pnpm@* ]]; then + echo "frontend packageManager must pin pnpm: ${package_manager}" >&2 + return 2 + fi + + frontend_corepack_home="$( + mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/ironclaw-corepack.XXXXXX" + )" + COREPACK_HOME="${frontend_corepack_home}" \ + corepack install --global "${package_manager}" + ( + cd "${repo_root}/crates/ironclaw_webui/frontend" + COREPACK_HOME="${frontend_corepack_home}" \ + corepack pnpm install --frozen-lockfile + ) +} + +run_frontend_tests() { + # Run from the package directory so Corepack honors its pinned packageManager + # version and its isolated setup cache without registry access in the guard. + COREPACK_HOME="${frontend_corepack_home}" \ + run bash -c 'cd "$1" && exec corepack pnpm test' \ + _ "${repo_root}/crates/ironclaw_webui/frontend" +} + case "${stage}" in self-test) "${repo_root}/scripts/ci/test-hermetic-test-process.sh" @@ -145,7 +185,8 @@ case "${stage}" in run "${repo_root}/scripts/reborn-e2e-rust.sh" "${1:-all}" ;; frontend) - run pnpm --dir crates/ironclaw_webui/frontend test + prepare_frontend_dependencies + run_frontend_tests ;; python-e2e) run_python_e2e @@ -163,7 +204,8 @@ case "${stage}" in run cargo test -p ironclaw_reborn_integration_tests \ --test reborn_qa_recorded_behavior -- --nocapture run "${repo_root}/scripts/reborn-e2e-rust.sh" all - run pnpm --dir crates/ironclaw_webui/frontend test + prepare_frontend_dependencies + run_frontend_tests run cargo build -p ironclaw --bin ironclaw run_python_e2e ;; diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index bd6ceb3dd02..db6869a8792 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -193,7 +193,7 @@ for workflow_contract in \ do workflow="${workflow_contract%%:*}" needle="${workflow_contract#*:}" - if ! rg -Fq "${needle}" "${repo_root}/${workflow}"; then + if ! grep -Fq "${needle}" "${repo_root}/${workflow}"; then echo "CI/local hermetic-suite parity lost: ${workflow} lacks '${needle}'" >&2 exit 1 fi @@ -208,7 +208,7 @@ for stage_call in \ "reborn-e2e-rust.sh" \ "run_python_e2e" do - if ! rg -Fq "${stage_call}" "${repo_root}/scripts/ci/run-hermetic-deterministic-suite.sh"; then + if ! grep -Fq "${stage_call}" "${repo_root}/scripts/ci/run-hermetic-deterministic-suite.sh"; then echo "canonical complete suite lost required stage: ${stage_call}" >&2 exit 1 fi From 9c2b1d7b1042762726541068c100def2b111f7a3 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 11:26:47 +0300 Subject: [PATCH 05/22] fix(ci): avoid races in network interposer lookup --- scripts/ci/hermetic-network-guard.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c index eb74a984f03..a0f9724cde9 100644 --- a/scripts/ci/hermetic-network-guard.c +++ b/scripts/ci/hermetic-network-guard.c @@ -18,27 +18,15 @@ typedef int (*connect_fn)(int, const struct sockaddr *, socklen_t); typedef ssize_t (*sendmsg_fn)(int, const struct msghdr *, int); typedef ssize_t (*sendto_fn)(int, const void *, size_t, int, const struct sockaddr *, socklen_t); static connect_fn real_connect(void) { - static connect_fn function; - if (function == NULL) { - function = (connect_fn)dlsym(RTLD_NEXT, "connect"); - } - return function; + return (connect_fn)dlsym(RTLD_NEXT, "connect"); } static sendmsg_fn real_sendmsg(void) { - static sendmsg_fn function; - if (function == NULL) { - function = (sendmsg_fn)dlsym(RTLD_NEXT, "sendmsg"); - } - return function; + return (sendmsg_fn)dlsym(RTLD_NEXT, "sendmsg"); } static sendto_fn real_sendto(void) { - static sendto_fn function; - if (function == NULL) { - function = (sendto_fn)dlsym(RTLD_NEXT, "sendto"); - } - return function; + return (sendto_fn)dlsym(RTLD_NEXT, "sendto"); } #endif From 74f193aef986c6245c3cfe750acdafc8bc45abcb Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 11:35:34 +0300 Subject: [PATCH 06/22] fix(ci): prefetch WebUI build toolchain for Cargo --- docs/internal/hermetic-deterministic-suite.md | 11 ++++++----- scripts/ci/run-hermetic-deterministic-suite.sh | 10 ++++++++-- scripts/ci/test-hermetic-test-process.sh | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md index 9a5f49e9880..3551641564b 100644 --- a/docs/internal/hermetic-deterministic-suite.md +++ b/docs/internal/hermetic-deterministic-suite.md @@ -18,11 +18,12 @@ compile-only jobs. Install the CI-pinned toolchains first. In particular, build the Emulate revision pinned in `.github/workflows/reborn-e2e.yml`, set -`IRONCLAW_EMULATE_CLI` to its built CLI, install the E2E Python package and -Chromium, install frontend dependencies, and prefetch locked Cargo -dependencies. Package installation is a build-tooling precondition. The -canonical script performs `cargo fetch --locked` before entering the guarded -process and then forces Cargo offline. Every command and descendant inside the +`IRONCLAW_EMULATE_CLI` to its built CLI, and install the E2E Python package and +Chromium. Package installation is a build-tooling precondition. The canonical +script performs `cargo fetch --locked` and installs the frontend's checked-in +`packageManager` into an isolated temporary Corepack cache before entering a +guarded Cargo stage. Cargo is then forced offline, and WebUI build scripts reuse +only that suite-owned Corepack cache. Every command and descendant inside the suite is guarded, so remote compiler wrappers are disabled rather than treated as a network exception. diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index 9db4cac6074..c858396f810 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -23,8 +23,10 @@ run() { prepare_rust_dependencies() { # Dependency acquisition is setup, not test behavior. Fetch once before the - # hermetic process switches Cargo into offline mode. + # hermetic process switches Cargo into offline mode. Rust builds can invoke + # the WebUI build script, so prepare its pinned package manager too. cargo fetch --locked + prepare_frontend_dependencies } run_root_partitions() { @@ -123,6 +125,10 @@ run_python_e2e() { prepare_frontend_dependencies() { local package_manager + if [[ -n "${frontend_corepack_home}" ]]; then + return + fi + package_manager="$( jq -r '.packageManager' \ "${repo_root}/crates/ironclaw_webui/frontend/package.json" @@ -137,6 +143,7 @@ prepare_frontend_dependencies() { )" COREPACK_HOME="${frontend_corepack_home}" \ corepack install --global "${package_manager}" + export COREPACK_HOME="${frontend_corepack_home}" ( cd "${repo_root}/crates/ironclaw_webui/frontend" COREPACK_HOME="${frontend_corepack_home}" \ @@ -204,7 +211,6 @@ case "${stage}" in run cargo test -p ironclaw_reborn_integration_tests \ --test reborn_qa_recorded_behavior -- --nocapture run "${repo_root}/scripts/reborn-e2e-rust.sh" all - prepare_frontend_dependencies run_frontend_tests run cargo build -p ironclaw --bin ironclaw run_python_e2e diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index db6869a8792..04c1972c799 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -203,6 +203,7 @@ for stage_call in \ "run_crate_tests" \ "run_root_partitions" \ "run_integration_tier" \ + "prepare_frontend_dependencies" \ "run-reborn-group-tests.sh" \ "check-reborn-qa-fixtures.sh" \ "reborn-e2e-rust.sh" \ From 93b0443006f63a78f2dd1e5d70ec248c7f13f7a3 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 11:53:18 +0300 Subject: [PATCH 07/22] test(ci): keep failure probes on loopback --- .../ironclaw_host_runtime/src/sandbox_process/connect.rs | 8 ++++++-- crates/ironclaw_skills/src/catalog.rs | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/ironclaw_host_runtime/src/sandbox_process/connect.rs b/crates/ironclaw_host_runtime/src/sandbox_process/connect.rs index e63f37547a3..4904ce1e512 100644 --- a/crates/ironclaw_host_runtime/src/sandbox_process/connect.rs +++ b/crates/ironclaw_host_runtime/src/sandbox_process/connect.rs @@ -526,15 +526,19 @@ mod tests { let _guard = lock_env(); set_runtime_env(DOCKER_HOST_ALLOW_REMOTE_ENV, "1"); + // User-info makes the policy parser classify this as non-loopback, + // while the actual connection remains on a closed loopback port so + // the test does not violate the hermetic test boundary. + let host = "http://remote-test@127.0.0.1:1"; let result = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .expect("build current-thread runtime for test") - .block_on(connect_override("http://203.0.113.5:2375")); + .block_on(connect_override(host)); remove_runtime_env(DOCKER_HOST_ALLOW_REMOTE_ENV); - let err = result.expect_err("203.0.113.5:2375 has nothing listening"); + let err = result.expect_err("127.0.0.1:1 has nothing listening"); let message = err.to_string(); assert!( !message.contains(DOCKER_HOST_ALLOW_REMOTE_ENV), diff --git a/crates/ironclaw_skills/src/catalog.rs b/crates/ironclaw_skills/src/catalog.rs index baecb9969e2..768db96b0c5 100644 --- a/crates/ironclaw_skills/src/catalog.rs +++ b/crates/ironclaw_skills/src/catalog.rs @@ -552,10 +552,9 @@ mod tests { #[tokio::test] async fn test_search_returns_error_on_network_failure() { - // Use RFC 5737 TEST-NET-1 (192.0.2.0/24) for reliable failure even behind proxies. - // Short timeout so the test doesn't block for the full 10s REQUEST_TIMEOUT. + // Use a closed loopback port so this failure-path test stays hermetic. let catalog = - SkillCatalog::with_url_and_timeout("http://192.0.2.1:9999", Duration::from_secs(1)); + SkillCatalog::with_url_and_timeout("http://127.0.0.1:1", Duration::from_secs(1)); let outcome = catalog.search("test").await; assert!(outcome.results.is_empty()); assert!(outcome.error.is_some()); From 8dbed3d9a915e169de8fbb778746f1e679483641 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 12:01:42 +0300 Subject: [PATCH 08/22] test(ci): isolate Copilot refresh failures --- crates/ironclaw_llm/src/github_copilot_auth.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/ironclaw_llm/src/github_copilot_auth.rs b/crates/ironclaw_llm/src/github_copilot_auth.rs index f2df59b1f04..691680fc967 100644 --- a/crates/ironclaw_llm/src/github_copilot_auth.rs +++ b/crates/ironclaw_llm/src/github_copilot_auth.rs @@ -646,6 +646,14 @@ mod tests { // --- CopilotTokenManager --- + fn loopback_failure_client() -> reqwest::Client { + let proxy = reqwest::Proxy::all("http://127.0.0.1:1").expect("valid loopback proxy"); + reqwest::Client::builder() + .proxy(proxy) + .build() + .expect("build test client") + } + #[tokio::test] async fn token_manager_caches_token_and_returns_same_value() { // Pre-populate the cache with a token that expires far in the future. @@ -691,7 +699,7 @@ mod tests { #[tokio::test] async fn token_manager_expired_token_triggers_refresh_path() { - let client = reqwest::Client::new(); + let client = loopback_failure_client(); let manager = CopilotTokenManager::new(client, "unused_oauth".to_string()); // Set a token that is already expired (expires_at in the past). @@ -715,7 +723,7 @@ mod tests { #[tokio::test] async fn token_manager_within_buffer_triggers_refresh() { - let client = reqwest::Client::new(); + let client = loopback_failure_client(); let manager = CopilotTokenManager::new(client, "unused_oauth".to_string()); // Set a token that expires within the refresh buffer window. From 8f351485e971e83b0e0c33e28bc6ecea0246e6af Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 12:09:02 +0300 Subject: [PATCH 09/22] fix(ci): preserve isolated Playwright toolchain --- .github/workflows/reborn-e2e.yml | 5 +- docs/internal/hermetic-deterministic-suite.md | 11 ++- scripts/ci/hermetic-network-guard.c | 84 +++++++++++++++++-- scripts/ci/hermetic-network-probe.c | 12 ++- scripts/ci/test-hermetic-test-process.sh | 35 ++++++++ 5 files changed, 137 insertions(+), 10 deletions(-) diff --git a/.github/workflows/reborn-e2e.yml b/.github/workflows/reborn-e2e.yml index a49d3763b95..92078782d05 100644 --- a/.github/workflows/reborn-e2e.yml +++ b/.github/workflows/reborn-e2e.yml @@ -160,6 +160,9 @@ jobs: ref: ${{ inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }} persist-credentials: false + - name: Configure isolated Playwright browser path + run: echo "PLAYWRIGHT_BROWSERS_PATH=${RUNNER_TEMP}/ms-playwright" >> "$GITHUB_ENV" + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -237,7 +240,7 @@ jobs: - name: Cache Playwright browsers uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: ~/.cache/ms-playwright + path: ${{ runner.temp }}/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('tests/e2e/pyproject.toml') }} restore-keys: ${{ runner.os }}-playwright- - name: Install E2E dependencies diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md index 3551641564b..837a50cfec8 100644 --- a/docs/internal/hermetic-deterministic-suite.md +++ b/docs/internal/hermetic-deterministic-suite.md @@ -19,7 +19,10 @@ compile-only jobs. Install the CI-pinned toolchains first. In particular, build the Emulate revision pinned in `.github/workflows/reborn-e2e.yml`, set `IRONCLAW_EMULATE_CLI` to its built CLI, and install the E2E Python package and -Chromium. Package installation is a build-tooling precondition. The canonical +Chromium with `PLAYWRIGHT_BROWSERS_PATH` set to a dedicated temporary tooling +directory. Keep that variable set when invoking the suite; CI caches, installs, +and reads Chromium through the same explicit path instead of the isolated test +home. Package installation is a build-tooling precondition. The canonical script performs `cargo fetch --locked` and installs the frontend's checked-in `packageManager` into an isolated temporary Corepack cache before entering a guarded Cargo stage. Cargo is then forced offline, and WebUI build scripts reuse @@ -37,7 +40,11 @@ Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. That boundar - denies non-loopback IP connections while permitting Unix sockets and deliberate IPv4/IPv6 localhost fakes. The syscall interposer records denied attempts on Linux and ordinary macOS binaries; macOS additionally uses the - process sandbox so SIP-protected launchers remain fail-closed. + process sandbox so SIP-protected launchers remain fail-closed. On Linux, + route-only UDP association is permitted because it sends no packet; explicit + and connected UDP sends remain blocked. This lets Chromium inspect a + route/source address without weakening the egress boundary. The coarser macOS + process sandbox may reject the harmless association as well. Rust wall-clock and random behavior is not overridden through process-global environment variables. Time-sensitive domain tests use their owning typed diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c index a0f9724cde9..c0cead6699e 100644 --- a/scripts/ci/hermetic-network-guard.c +++ b/scripts/ci/hermetic-network-guard.c @@ -15,12 +15,17 @@ #if !defined(__APPLE__) typedef int (*connect_fn)(int, const struct sockaddr *, socklen_t); +typedef ssize_t (*send_fn)(int, const void *, size_t, int); typedef ssize_t (*sendmsg_fn)(int, const struct msghdr *, int); typedef ssize_t (*sendto_fn)(int, const void *, size_t, int, const struct sockaddr *, socklen_t); static connect_fn real_connect(void) { return (connect_fn)dlsym(RTLD_NEXT, "connect"); } +static send_fn real_send(void) { + return (send_fn)dlsym(RTLD_NEXT, "send"); +} + static sendmsg_fn real_sendmsg(void) { return (sendmsg_fn)dlsym(RTLD_NEXT, "sendmsg"); } @@ -46,6 +51,34 @@ static int is_loopback(const struct sockaddr *address) { return 1; } +static int socket_is_datagram(int socket_fd) { + int socket_type = 0; + socklen_t length = sizeof(socket_type); + int saved_errno = errno; + int status = getsockopt(socket_fd, SOL_SOCKET, SO_TYPE, &socket_type, &length); + errno = saved_errno; + return status == 0 && socket_type == SOCK_DGRAM; +} + +static const struct sockaddr *non_loopback_destination( + int socket_fd, + const struct sockaddr *address, + struct sockaddr_storage *peer +) { + if (address != NULL) { + return is_loopback(address) ? NULL : address; + } + + socklen_t length = sizeof(*peer); + int saved_errno = errno; + int status = getpeername(socket_fd, (struct sockaddr *)peer, &length); + errno = saved_errno; + if (status == 0 && !is_loopback((const struct sockaddr *)peer)) { + return (const struct sockaddr *)peer; + } + return NULL; +} + static void write_violation(int fd, const char *message, size_t length) { size_t offset = 0; while (offset < length) { @@ -105,7 +138,11 @@ static int guarded_connect( const struct sockaddr *address, socklen_t address_length ) { - if (!is_loopback(address)) { + /* + * UDP connect() sends no packet; Chromium uses it to inspect route/source + * selection. Actual connected UDP writes are rejected by the send guards. + */ + if (!is_loopback(address) && !socket_is_datagram(socket_fd)) { record_violation(address); errno = EPERM; return -1; @@ -122,11 +159,40 @@ static int guarded_connect( #endif } +static ssize_t guarded_send( + int socket_fd, + const void *buffer, + size_t length, + int flags +) { + struct sockaddr_storage peer; + const struct sockaddr *blocked = + non_loopback_destination(socket_fd, NULL, &peer); + if (blocked != NULL) { + record_violation(blocked); + errno = EPERM; + return -1; + } +#if defined(__APPLE__) + return (ssize_t)syscall(SYS_sendto, socket_fd, buffer, length, flags, NULL, 0); +#else + send_fn function = real_send(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(socket_fd, buffer, length, flags); +#endif +} + static ssize_t guarded_sendmsg(int socket_fd, const struct msghdr *message, int flags) { const struct sockaddr *address = message == NULL ? NULL : (const struct sockaddr *)message->msg_name; - if (!is_loopback(address)) { - record_violation(address); + struct sockaddr_storage peer; + const struct sockaddr *blocked = + non_loopback_destination(socket_fd, address, &peer); + if (blocked != NULL) { + record_violation(blocked); errno = EPERM; return -1; } @@ -150,8 +216,11 @@ static ssize_t guarded_sendto( const struct sockaddr *address, socklen_t address_length ) { - if (!is_loopback(address)) { - record_violation(address); + struct sockaddr_storage peer; + const struct sockaddr *blocked = + non_loopback_destination(socket_fd, address, &peer); + if (blocked != NULL) { + record_violation(blocked); errno = EPERM; return -1; } @@ -186,6 +255,7 @@ static ssize_t guarded_sendto( } DYLD_INTERPOSE(guarded_connect, connect); +DYLD_INTERPOSE(guarded_send, send); DYLD_INTERPOSE(guarded_sendmsg, sendmsg); DYLD_INTERPOSE(guarded_sendto, sendto); #else @@ -193,6 +263,10 @@ int connect(int socket_fd, const struct sockaddr *address, socklen_t address_len return guarded_connect(socket_fd, address, address_length); } +ssize_t send(int socket_fd, const void *buffer, size_t length, int flags) { + return guarded_send(socket_fd, buffer, length, flags); +} + ssize_t sendmsg(int socket_fd, const struct msghdr *message, int flags) { return guarded_sendmsg(socket_fd, message, flags); } diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c index 82e8a10c302..d871bcd9d80 100644 --- a/scripts/ci/hermetic-network-probe.c +++ b/scripts/ci/hermetic-network-probe.c @@ -12,7 +12,11 @@ int main(int argc, char **argv) { return 2; } - int use_udp = argc == 3 && strcmp(argv[2], "udp") == 0; + const char *mode = argc == 3 ? argv[2] : "tcp"; + int use_udp = + strcmp(mode, "udp") == 0 + || strcmp(mode, "udp-connected") == 0 + || strcmp(mode, "udp-connect-only") == 0; int fd = socket(AF_INET, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); if (fd < 0) { return 3; @@ -31,7 +35,7 @@ int main(int argc, char **argv) { return 4; } int network_status; - if (use_udp) { + if (strcmp(mode, "udp") == 0) { const char byte = 'x'; network_status = (int)sendto( fd, @@ -43,6 +47,10 @@ int main(int argc, char **argv) { ); } else { network_status = connect(fd, (const struct sockaddr *)&address, sizeof(address)); + if (network_status == 0 && strcmp(mode, "udp-connected") == 0) { + const char byte = 'x'; + network_status = (int)send(fd, &byte, sizeof(byte), 0); + } } if (network_status < 0 && errno == EPERM) { fprintf( diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 04c1972c799..c8ba3b6475c 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -22,6 +22,7 @@ run_probe() { GOOGLE_APPLICATION_CREDENTIALS="/developer/credential.json" \ LLM_BACKEND="ambient-provider" \ REBORN_TOOL_DISCLOSURE="Bridged" \ + PLAYWRIGHT_BROWSERS_PATH="${probe_dir}/playwright-browsers" \ IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ "${runner}" -- bash -c ' set -euo pipefail @@ -63,6 +64,10 @@ run_probe() { [[ "${TZ}" == "UTC" ]] [[ "${LANG}" == "C.UTF-8" ]] [[ "${LC_ALL}" == "C.UTF-8" ]] + if [[ "${PLAYWRIGHT_BROWSERS_PATH:-}" != */playwright-browsers ]]; then + echo "explicit Playwright browser toolchain path was not preserved" >&2 + exit 35 + fi if [[ "${PYTHONHASHSEED:-}" != "0" ]]; then echo "deterministic Python hash seed is not injected" >&2 exit 34 @@ -140,6 +145,36 @@ if [[ "${udp_status}" -eq 0 || "${udp_output}" != *"non-loopback network attempt exit 1 fi +# UDP connect() only selects a route and sends no packet, so browser and PAC +# source-address inspection remains usable. A subsequent connected send is +# still external I/O and must fail loudly. +if [[ "$(uname -s)" == "Linux" ]]; then + "${runner}" -- "${network_probe}" 192.0.2.1 udp-connect-only +else + # The macOS process sandbox may reject even a route-only association. That is + # safe; unlike the interposer it cannot distinguish association from I/O. + set +e + "${runner}" -- "${network_probe}" 192.0.2.1 udp-connect-only >/dev/null 2>&1 + route_probe_status=$? + set -e + if [[ "${route_probe_status}" -ne 0 && "${route_probe_status}" -ne 90 ]]; then + echo "unexpected macOS UDP route-probe result: ${route_probe_status}" >&2 + exit 1 + fi +fi +set +e +connected_udp_output="$( + IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ + "${runner}" -- "${network_probe}" 192.0.2.1 udp-connected 2>&1 +)" +connected_udp_status=$? +set -e +if [[ "${connected_udp_status}" -eq 0 || "${connected_udp_output}" != *"non-loopback network attempt"* ]]; then + echo "unexpected connected non-loopback UDP send was not reported" >&2 + printf '%s\n' "${connected_udp_output}" >&2 + exit 1 +fi + # Deliberate localhost fakes remain usable. A refused port is sufficient: the # guard must allow the connect syscall to reach the kernel rather than report it. "${runner}" -- "${network_probe}" 127.0.0.1 From 96b8aa754bc5f6434ed65cabf5174603af2f7642 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 13:59:36 +0300 Subject: [PATCH 10/22] fix(ci): close connected UDP write bypasses --- docs/internal/hermetic-deterministic-suite.md | 7 +- scripts/ci/hermetic-network-guard.c | 111 +++++++++++++++++- scripts/ci/hermetic-network-probe.c | 31 +++++ scripts/ci/test-hermetic-test-process.sh | 28 +++-- 4 files changed, 162 insertions(+), 15 deletions(-) diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md index 837a50cfec8..34a8c38035f 100644 --- a/docs/internal/hermetic-deterministic-suite.md +++ b/docs/internal/hermetic-deterministic-suite.md @@ -42,9 +42,10 @@ Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. That boundar attempts on Linux and ordinary macOS binaries; macOS additionally uses the process sandbox so SIP-protected launchers remain fail-closed. On Linux, route-only UDP association is permitted because it sends no packet; explicit - and connected UDP sends remain blocked. This lets Chromium inspect a - route/source address without weakening the egress boundary. The coarser macOS - process sandbox may reject the harmless association as well. + and connected UDP transmission through `send`, `sendto`, `sendmsg`, + `sendmmsg`, `write`, and `writev` remains blocked. This lets Chromium inspect + a route/source address without weakening the egress boundary. The coarser + macOS process sandbox may reject the harmless association as well. Rust wall-clock and random behavior is not overridden through process-global environment variables. Time-sensitive domain tests use their owning typed diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c index c0cead6699e..868707563ed 100644 --- a/scripts/ci/hermetic-network-guard.c +++ b/scripts/ci/hermetic-network-guard.c @@ -11,13 +11,17 @@ #include #include #include +#include #include #if !defined(__APPLE__) typedef int (*connect_fn)(int, const struct sockaddr *, socklen_t); typedef ssize_t (*send_fn)(int, const void *, size_t, int); +typedef int (*sendmmsg_fn)(int, struct mmsghdr *, unsigned int, int); typedef ssize_t (*sendmsg_fn)(int, const struct msghdr *, int); typedef ssize_t (*sendto_fn)(int, const void *, size_t, int, const struct sockaddr *, socklen_t); +typedef ssize_t (*write_fn)(int, const void *, size_t); +typedef ssize_t (*writev_fn)(int, const struct iovec *, int); static connect_fn real_connect(void) { return (connect_fn)dlsym(RTLD_NEXT, "connect"); } @@ -26,6 +30,10 @@ static send_fn real_send(void) { return (send_fn)dlsym(RTLD_NEXT, "send"); } +static sendmmsg_fn real_sendmmsg(void) { + return (sendmmsg_fn)dlsym(RTLD_NEXT, "sendmmsg"); +} + static sendmsg_fn real_sendmsg(void) { return (sendmsg_fn)dlsym(RTLD_NEXT, "sendmsg"); } @@ -33,6 +41,14 @@ static sendmsg_fn real_sendmsg(void) { static sendto_fn real_sendto(void) { return (sendto_fn)dlsym(RTLD_NEXT, "sendto"); } + +static write_fn real_write(void) { + return (write_fn)dlsym(RTLD_NEXT, "write"); +} + +static writev_fn real_writev(void) { + return (writev_fn)dlsym(RTLD_NEXT, "writev"); +} #endif static int is_loopback(const struct sockaddr *address) { @@ -82,7 +98,8 @@ static const struct sockaddr *non_loopback_destination( static void write_violation(int fd, const char *message, size_t length) { size_t offset = 0; while (offset < length) { - ssize_t written = write(fd, message + offset, length - offset); + ssize_t written = + (ssize_t)syscall(SYS_write, fd, message + offset, length - offset); if (written > 0) { offset += (size_t)written; continue; @@ -185,6 +202,35 @@ static ssize_t guarded_send( #endif } +#if !defined(__APPLE__) +static int guarded_sendmmsg( + int socket_fd, + struct mmsghdr *messages, + unsigned int message_count, + int flags +) { + for (unsigned int index = 0; index < message_count; index++) { + const struct sockaddr *address = + (const struct sockaddr *)messages[index].msg_hdr.msg_name; + struct sockaddr_storage peer; + const struct sockaddr *blocked = + non_loopback_destination(socket_fd, address, &peer); + if (blocked != NULL) { + record_violation(blocked); + errno = EPERM; + return -1; + } + } + + sendmmsg_fn function = real_sendmmsg(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(socket_fd, messages, message_count, flags); +} +#endif + static ssize_t guarded_sendmsg(int socket_fd, const struct msghdr *message, int flags) { const struct sockaddr *address = message == NULL ? NULL : (const struct sockaddr *)message->msg_name; @@ -244,6 +290,50 @@ static ssize_t guarded_sendto( #endif } +static ssize_t guarded_write(int fd, const void *buffer, size_t length) { + struct sockaddr_storage peer; + const struct sockaddr *blocked = non_loopback_destination(fd, NULL, &peer); + if (blocked != NULL) { + record_violation(blocked); + errno = EPERM; + return -1; + } +#if defined(__APPLE__) + return (ssize_t)syscall(SYS_write, fd, buffer, length); +#else + write_fn function = real_write(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(fd, buffer, length); +#endif +} + +static ssize_t guarded_writev( + int fd, + const struct iovec *iovecs, + int iovec_count +) { + struct sockaddr_storage peer; + const struct sockaddr *blocked = non_loopback_destination(fd, NULL, &peer); + if (blocked != NULL) { + record_violation(blocked); + errno = EPERM; + return -1; + } +#if defined(__APPLE__) + return (ssize_t)syscall(SYS_writev, fd, iovecs, iovec_count); +#else + writev_fn function = real_writev(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(fd, iovecs, iovec_count); +#endif +} + #if defined(__APPLE__) #define DYLD_INTERPOSE(replacement, replacee) \ __attribute__((used)) static struct { \ @@ -258,6 +348,8 @@ DYLD_INTERPOSE(guarded_connect, connect); DYLD_INTERPOSE(guarded_send, send); DYLD_INTERPOSE(guarded_sendmsg, sendmsg); DYLD_INTERPOSE(guarded_sendto, sendto); +DYLD_INTERPOSE(guarded_write, write); +DYLD_INTERPOSE(guarded_writev, writev); #else int connect(int socket_fd, const struct sockaddr *address, socklen_t address_length) { return guarded_connect(socket_fd, address, address_length); @@ -267,6 +359,15 @@ ssize_t send(int socket_fd, const void *buffer, size_t length, int flags) { return guarded_send(socket_fd, buffer, length, flags); } +int sendmmsg( + int socket_fd, + struct mmsghdr *messages, + unsigned int message_count, + int flags +) { + return guarded_sendmmsg(socket_fd, messages, message_count, flags); +} + ssize_t sendmsg(int socket_fd, const struct msghdr *message, int flags) { return guarded_sendmsg(socket_fd, message, flags); } @@ -281,4 +382,12 @@ ssize_t sendto( ) { return guarded_sendto(socket_fd, buffer, length, flags, address, address_length); } + +ssize_t write(int fd, const void *buffer, size_t length) { + return guarded_write(fd, buffer, length); +} + +ssize_t writev(int fd, const struct iovec *iovecs, int iovec_count) { + return guarded_writev(fd, iovecs, iovec_count); +} #endif diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c index d871bcd9d80..c537dad11c3 100644 --- a/scripts/ci/hermetic-network-probe.c +++ b/scripts/ci/hermetic-network-probe.c @@ -5,6 +5,7 @@ #include #include #include +#include #include int main(int argc, char **argv) { @@ -17,6 +18,13 @@ int main(int argc, char **argv) { strcmp(mode, "udp") == 0 || strcmp(mode, "udp-connected") == 0 || strcmp(mode, "udp-connect-only") == 0; +#if defined(__linux__) + use_udp = use_udp || strcmp(mode, "udp-sendmmsg") == 0; +#endif + use_udp = + use_udp + || strcmp(mode, "udp-write") == 0 + || strcmp(mode, "udp-writev") == 0; int fd = socket(AF_INET, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); if (fd < 0) { return 3; @@ -50,6 +58,29 @@ int main(int argc, char **argv) { if (network_status == 0 && strcmp(mode, "udp-connected") == 0) { const char byte = 'x'; network_status = (int)send(fd, &byte, sizeof(byte), 0); + } else if (network_status == 0 && strcmp(mode, "udp-write") == 0) { + const char byte = 'x'; + network_status = (int)write(fd, &byte, sizeof(byte)); + } else if (network_status == 0 && strcmp(mode, "udp-writev") == 0) { + char byte = 'x'; + const struct iovec iovec = { + .iov_base = &byte, + .iov_len = sizeof(byte), + }; + network_status = (int)writev(fd, &iovec, 1); +#if defined(__linux__) + } else if (network_status == 0 && strcmp(mode, "udp-sendmmsg") == 0) { + char byte = 'x'; + struct iovec iovec = { + .iov_base = &byte, + .iov_len = sizeof(byte), + }; + struct mmsghdr message; + memset(&message, 0, sizeof(message)); + message.msg_hdr.msg_iov = &iovec; + message.msg_hdr.msg_iovlen = 1; + network_status = sendmmsg(fd, &message, 1, 0); +#endif } } if (network_status < 0 && errno == EPERM) { diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index c8ba3b6475c..685606e957e 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -162,18 +162,24 @@ else exit 1 fi fi -set +e -connected_udp_output="$( - IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ - "${runner}" -- "${network_probe}" 192.0.2.1 udp-connected 2>&1 -)" -connected_udp_status=$? -set -e -if [[ "${connected_udp_status}" -eq 0 || "${connected_udp_output}" != *"non-loopback network attempt"* ]]; then - echo "unexpected connected non-loopback UDP send was not reported" >&2 - printf '%s\n' "${connected_udp_output}" >&2 - exit 1 +connected_udp_modes=(udp-connected udp-write udp-writev) +if [[ "$(uname -s)" == "Linux" ]]; then + connected_udp_modes+=(udp-sendmmsg) fi +for connected_udp_mode in "${connected_udp_modes[@]}"; do + set +e + connected_udp_output="$( + IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ + "${runner}" -- "${network_probe}" 192.0.2.1 "${connected_udp_mode}" 2>&1 + )" + connected_udp_status=$? + set -e + if [[ "${connected_udp_status}" -eq 0 || "${connected_udp_output}" != *"non-loopback network attempt"* ]]; then + echo "unexpected ${connected_udp_mode} non-loopback UDP send was not reported" >&2 + printf '%s\n' "${connected_udp_output}" >&2 + exit 1 + fi +done # Deliberate localhost fakes remain usable. A refused port is sufficient: the # guard must allow the connect syscall to reach the kernel rather than report it. From c76b65436b41769b8a4a0fa3a043c161b1f61f67 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 14:04:33 +0300 Subject: [PATCH 11/22] fix(ci): expose Linux sendmmsg probe API --- scripts/ci/hermetic-network-probe.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c index c537dad11c3..dfbfe91593c 100644 --- a/scripts/ci/hermetic-network-probe.c +++ b/scripts/ci/hermetic-network-probe.c @@ -1,3 +1,5 @@ +#define _GNU_SOURCE + #include #include #include From 31ca9dc75b25a86adf1d86b733d41014b26660e5 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 14:38:53 +0300 Subject: [PATCH 12/22] fix(ci): allow IPv4-mapped loopback fakes --- scripts/ci/hermetic-network-guard.c | 6 ++++- scripts/ci/hermetic-network-probe.c | 34 +++++++++++++++++------- scripts/ci/test-hermetic-test-process.sh | 13 +++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c index 868707563ed..3f40e261133 100644 --- a/scripts/ci/hermetic-network-guard.c +++ b/scripts/ci/hermetic-network-guard.c @@ -61,7 +61,11 @@ static int is_loopback(const struct sockaddr *address) { } if (address->sa_family == AF_INET6) { const struct sockaddr_in6 *ipv6 = (const struct sockaddr_in6 *)address; - return IN6_IS_ADDR_LOOPBACK(&ipv6->sin6_addr); + return IN6_IS_ADDR_LOOPBACK(&ipv6->sin6_addr) + || ( + IN6_IS_ADDR_V4MAPPED(&ipv6->sin6_addr) + && ipv6->sin6_addr.s6_addr[12] == 127 + ); } /* Unix sockets and non-IP kernel transports are local by definition. */ return 1; diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c index dfbfe91593c..0faad08228c 100644 --- a/scripts/ci/hermetic-network-probe.c +++ b/scripts/ci/hermetic-network-probe.c @@ -27,7 +27,8 @@ int main(int argc, char **argv) { use_udp || strcmp(mode, "udp-write") == 0 || strcmp(mode, "udp-writev") == 0; - int fd = socket(AF_INET, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + int family = strchr(argv[1], ':') == NULL ? AF_INET : AF_INET6; + int fd = socket(family, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); if (fd < 0) { return 3; } @@ -36,13 +37,27 @@ int main(int argc, char **argv) { close(fd); return 5; } - struct sockaddr_in address; + struct sockaddr_storage address; memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(9); - if (inet_pton(AF_INET, argv[1], &address.sin_addr) != 1) { - close(fd); - return 4; + socklen_t address_length; + if (family == AF_INET) { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)&address; + ipv4->sin_family = AF_INET; + ipv4->sin_port = htons(9); + address_length = sizeof(*ipv4); + if (inet_pton(AF_INET, argv[1], &ipv4->sin_addr) != 1) { + close(fd); + return 4; + } + } else { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&address; + ipv6->sin6_family = AF_INET6; + ipv6->sin6_port = htons(9); + address_length = sizeof(*ipv6); + if (inet_pton(AF_INET6, argv[1], &ipv6->sin6_addr) != 1) { + close(fd); + return 4; + } } int network_status; if (strcmp(mode, "udp") == 0) { @@ -53,10 +68,11 @@ int main(int argc, char **argv) { sizeof(byte), 0, (const struct sockaddr *)&address, - sizeof(address) + address_length ); } else { - network_status = connect(fd, (const struct sockaddr *)&address, sizeof(address)); + network_status = + connect(fd, (const struct sockaddr *)&address, address_length); if (network_status == 0 && strcmp(mode, "udp-connected") == 0) { const char byte = 'x'; network_status = (int)send(fd, &byte, sizeof(byte), 0); diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 685606e957e..7a74b54114f 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -184,6 +184,19 @@ done # Deliberate localhost fakes remain usable. A refused port is sufficient: the # guard must allow the connect syscall to reach the kernel rather than report it. "${runner}" -- "${network_probe}" 127.0.0.1 +if [[ "$(uname -s)" == "Linux" ]]; then + "${runner}" -- "${network_probe}" ::ffff:127.0.0.1 +else + # sandbox-exec may reject IPv4-mapped IPv6 before the interposer observes it. + set +e + "${runner}" -- "${network_probe}" ::ffff:127.0.0.1 >/dev/null 2>&1 + mapped_loopback_status=$? + set -e + if [[ "${mapped_loopback_status}" -ne 0 && "${mapped_loopback_status}" -ne 90 ]]; then + echo "unexpected macOS IPv4-mapped loopback result: ${mapped_loopback_status}" >&2 + exit 1 + fi +fi "${runner}" -- python3 - "${repo_root}/tests/e2e" <<'PY' import sys From 31d78122a73abee072d429a762f669f384f372cc Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 15:44:02 +0300 Subject: [PATCH 13/22] fix(ci): preserve guard and one-time setup --- .github/workflows/reborn-e2e.yml | 4 +++ .github/workflows/reborn-tests.yml | 24 +++++++++----- docs/internal/hermetic-deterministic-suite.md | 10 ++++++ .../ci/run-hermetic-deterministic-suite.sh | 20 +++++++++-- scripts/ci/test-hermetic-test-process.sh | 33 ++++++++++++++++--- tests/e2e/hermetic_process.py | 15 +++++++-- 6 files changed, 87 insertions(+), 19 deletions(-) diff --git a/.github/workflows/reborn-e2e.yml b/.github/workflows/reborn-e2e.yml index 92078782d05..541987538c1 100644 --- a/.github/workflows/reborn-e2e.yml +++ b/.github/workflows/reborn-e2e.yml @@ -150,6 +150,8 @@ jobs: if: needs.changes.outputs.has_e2e_scope == 'true' runs-on: ubuntu-latest timeout-minutes: 30 + env: + IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE: "1" steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -407,6 +409,8 @@ jobs: if: needs.changes.outputs.has_e2e_scope == 'true' runs-on: ubuntu-latest timeout-minutes: 30 + env: + IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE: "1" steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index f5ffcf09dd2..43f549e9d82 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -347,6 +347,8 @@ jobs: echo "Running Reborn crate bucket: ${BUCKET_NAME}" printf '%s\n' "${BUCKET_PACKAGES}" | jq -r '.[] | "- " + .' + scripts/ci/run-hermetic-deterministic-suite.sh prepare-command + export IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1 mkdir -p coverage while IFS= read -r package; do @@ -666,7 +668,9 @@ jobs: # instrumented binaries) keeps the build-cache speed win while # guaranteeing this lane's lcov reflects only its own test run. cargo llvm-cov clean --profraw-only - scripts/ci/run-hermetic-deterministic-suite.sh command \ + scripts/ci/run-hermetic-deterministic-suite.sh prepare-command + IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1 \ + scripts/ci/run-hermetic-deterministic-suite.sh command \ scripts/ci/reborn-coverage-lane-run.sh part-${{ matrix.lane }}.lcov - name: Upload lane lcov artifact @@ -868,14 +872,16 @@ jobs: run: scripts/ci/check-reborn-qa-fixtures.sh - name: Run Reborn QA fixture contracts and replay - run: >- - scripts/ci/run-hermetic-deterministic-suite.sh - command - cargo test - -p ironclaw_reborn_integration_tests - --test reborn_qa_recorded_behavior - -- - --nocapture + run: | + scripts/ci/run-hermetic-deterministic-suite.sh prepare-command + IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1 \ + scripts/ci/run-hermetic-deterministic-suite.sh \ + command \ + cargo test \ + -p ironclaw_reborn_integration_tests \ + --test reborn_qa_recorded_behavior \ + -- \ + --nocapture reborn-tests: name: Tests (Reborn) diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md index 34a8c38035f..d82c111b147 100644 --- a/docs/internal/hermetic-deterministic-suite.md +++ b/docs/internal/hermetic-deterministic-suite.md @@ -60,6 +60,16 @@ scripts/ci/run-hermetic-deterministic-suite.sh rust-e2e substrates scripts/ci/run-hermetic-deterministic-suite.sh command cargo test -p ironclaw_network ``` +`command` prepares locked Rust and WebUI dependencies by default for a +standalone local invocation. CI loops that already installed any required +WebUI dependencies prepare Cargo once, then skip repeated setup: + +```bash +scripts/ci/run-hermetic-deterministic-suite.sh prepare-command +export IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1 +scripts/ci/run-hermetic-deterministic-suite.sh command cargo test -p ironclaw_network +``` + The guard is mutation-tested by: ```bash diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index c858396f810..695c0248d1b 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -25,10 +25,14 @@ prepare_rust_dependencies() { # Dependency acquisition is setup, not test behavior. Fetch once before the # hermetic process switches Cargo into offline mode. Rust builds can invoke # the WebUI build script, so prepare its pinned package manager too. - cargo fetch --locked + prepare_command_dependencies prepare_frontend_dependencies } +prepare_command_dependencies() { + cargo fetch --locked +} + run_root_partitions() { local partition for partition in 0 1 2 3; do @@ -198,6 +202,9 @@ case "${stage}" in python-e2e) run_python_e2e ;; + prepare-command) + prepare_command_dependencies + ;; all) "${repo_root}/scripts/ci/test-hermetic-test-process.sh" prepare_rust_dependencies @@ -220,12 +227,19 @@ case "${stage}" in echo "command stage requires a command" >&2 exit 2 fi - prepare_rust_dependencies + case "${IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE:-0}" in + 0) prepare_rust_dependencies ;; + 1) ;; + *) + echo "IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE must be 0 or 1" >&2 + exit 2 + ;; + esac run "$@" ;; *) echo "unknown hermetic deterministic-suite stage: ${stage}" >&2 - echo "expected: all, self-test, crates, root, groups, integration, qa, rust-e2e, frontend, python-e2e, command" >&2 + echo "expected: all, self-test, crates, root, groups, integration, qa, rust-e2e, frontend, python-e2e, prepare-command, command" >&2 exit 2 ;; esac diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 7a74b54114f..772350507e6 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -101,6 +101,19 @@ if [[ "${parallel_root_count}" != "4" ]]; then exit 1 fi +no_prepare_bin="${probe_dir}/no-prepare-bin" +mkdir -p "${no_prepare_bin}" +cat > "${no_prepare_bin}/cargo" <<'SH' +#!/usr/bin/env bash +echo "command stage unexpectedly repeated dependency preparation" >&2 +exit 97 +SH +chmod +x "${no_prepare_bin}/cargo" +PATH="${no_prepare_bin}:${PATH}" \ + IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1 \ + "${repo_root}/scripts/ci/run-hermetic-deterministic-suite.sh" \ + command bash -c 'test -n "${IRONCLAW_HERMETIC_ROOT:-}"' + set +e network_output="$( IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ @@ -205,16 +218,22 @@ sys.path.insert(0, sys.argv[1]) from hermetic_process import forward_hermetic_process_env source = { + "IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY": "/tmp/guard.so", "IRONCLAW_HERMETIC_NETWORK_VIOLATIONS": "/tmp/violations", - "LD_PRELOAD": "/tmp/guard.so", + "LD_PRELOAD": "/tmp/existing.so", + "DYLD_INSERT_LIBRARIES": "/tmp/existing.dylib", "ANTHROPIC_API_KEY": "must-not-forward", } child = {} forward_hermetic_process_env(child, source) -assert child == { - "IRONCLAW_HERMETIC_NETWORK_VIOLATIONS": "/tmp/violations", - "LD_PRELOAD": "/tmp/guard.so", -} +assert child["IRONCLAW_HERMETIC_NETWORK_VIOLATIONS"] == "/tmp/violations" +if sys.platform == "darwin": + assert child["DYLD_INSERT_LIBRARIES"] == "/tmp/guard.so:/tmp/existing.dylib" + assert child["LD_PRELOAD"] == "/tmp/existing.so" + assert child["DYLD_FORCE_FLAT_NAMESPACE"] == "1" +elif sys.platform.startswith("linux"): + assert child["LD_PRELOAD"] == "/tmp/guard.so:/tmp/existing.so" + assert child["DYLD_INSERT_LIBRARIES"] == "/tmp/existing.dylib" PY set +e @@ -242,7 +261,10 @@ fi for workflow_contract in \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh groups" \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh command" \ + ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh prepare-command" \ + ".github/workflows/reborn-tests.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1" \ ".github/workflows/reborn-e2e.yml:scripts/ci/run-hermetic-deterministic-suite.sh" \ + ".github/workflows/reborn-e2e.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE" \ ".github/workflows/code_style.yml:scripts/ci/test-hermetic-test-process.sh" do workflow="${workflow_contract%%:*}" @@ -254,6 +276,7 @@ do done for stage_call in \ + "prepare_command_dependencies" \ "run_crate_tests" \ "run_root_partitions" \ "run_integration_tier" \ diff --git a/tests/e2e/hermetic_process.py b/tests/e2e/hermetic_process.py index e7a3feeb7c1..dd6f7490f69 100644 --- a/tests/e2e/hermetic_process.py +++ b/tests/e2e/hermetic_process.py @@ -30,7 +30,18 @@ def forward_hermetic_process_env( if guard_library is None: return if sys.platform == "darwin": - env.setdefault("DYLD_INSERT_LIBRARIES", guard_library) + _prepend_preload(env, "DYLD_INSERT_LIBRARIES", guard_library) env.setdefault("DYLD_FORCE_FLAT_NAMESPACE", "1") elif sys.platform.startswith("linux"): - env.setdefault("LD_PRELOAD", guard_library) + _prepend_preload(env, "LD_PRELOAD", guard_library) + + +def _prepend_preload( + env: MutableMapping[str, str], + key: str, + guard_library: str, +) -> None: + libraries = [value for value in env.get(key, "").split(":") if value] + if guard_library not in libraries: + libraries.insert(0, guard_library) + env[key] = ":".join(libraries) From f3c5e3d1ae16d73d55919d424483d3d861446ba2 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 15:53:57 +0300 Subject: [PATCH 14/22] fix(ci): prepare coverage WebUI inputs once --- .github/workflows/reborn-tests.yml | 18 ++++++++++++++++++ scripts/ci/test-hermetic-test-process.sh | 1 + 2 files changed, 19 insertions(+) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index 43f549e9d82..92d2db63a46 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -620,6 +620,24 @@ jobs: with: components: llvm-tools-preview + - name: Enable pnpm for setup-node cache + run: corepack enable pnpm + + - name: Install Node.js for instrumented WebUI bundle builds + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 + with: + node-version: "22" + cache: "pnpm" + cache-dependency-path: crates/ironclaw_webui/frontend/pnpm-lock.yaml + + - name: Enable pnpm + run: corepack enable pnpm + + - name: Install WebUI frontend dependencies for coverage + run: | + cd crates/ironclaw_webui/frontend + pnpm install --frozen-lockfile + - name: Install mold and clang run: | scripts/ci/install-ci-apt-packages.sh clang mold diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 772350507e6..949714f48e9 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -263,6 +263,7 @@ for workflow_contract in \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh command" \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh prepare-command" \ ".github/workflows/reborn-tests.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1" \ + ".github/workflows/reborn-tests.yml:Install WebUI frontend dependencies for coverage" \ ".github/workflows/reborn-e2e.yml:scripts/ci/run-hermetic-deterministic-suite.sh" \ ".github/workflows/reborn-e2e.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE" \ ".github/workflows/code_style.yml:scripts/ci/test-hermetic-test-process.sh" From da1a280e975ab631646b4311e302fa0c539a3e33 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 15:58:30 +0300 Subject: [PATCH 15/22] fix(ci): prepare QA WebUI inputs once --- .github/workflows/reborn-tests.yml | 18 ++++++++++++++++++ scripts/ci/test-hermetic-test-process.sh | 1 + 2 files changed, 19 insertions(+) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index 92d2db63a46..bd737981e63 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -870,6 +870,24 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Enable pnpm for setup-node cache + run: corepack enable pnpm + + - name: Install Node.js for QA replay WebUI bundle builds + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 + with: + node-version: "22" + cache: "pnpm" + cache-dependency-path: crates/ironclaw_webui/frontend/pnpm-lock.yaml + + - name: Enable pnpm + run: corepack enable pnpm + + - name: Install WebUI frontend dependencies for QA replay + run: | + cd crates/ironclaw_webui/frontend + pnpm install --frozen-lockfile + - name: Restore Rust cache uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 949714f48e9..fdce69ee0fb 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -264,6 +264,7 @@ for workflow_contract in \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh prepare-command" \ ".github/workflows/reborn-tests.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1" \ ".github/workflows/reborn-tests.yml:Install WebUI frontend dependencies for coverage" \ + ".github/workflows/reborn-tests.yml:Install WebUI frontend dependencies for QA replay" \ ".github/workflows/reborn-e2e.yml:scripts/ci/run-hermetic-deterministic-suite.sh" \ ".github/workflows/reborn-e2e.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE" \ ".github/workflows/code_style.yml:scripts/ci/test-hermetic-test-process.sh" From 556ab89d872b805632fbaec1ddf404acbedf8996 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 16:03:46 +0300 Subject: [PATCH 16/22] fix(ci): preserve prepared Corepack cache --- .github/workflows/reborn-tests.yml | 9 +++++++++ scripts/ci/test-hermetic-test-process.sh | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index bd737981e63..1d7c6118801 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -239,6 +239,9 @@ jobs: ref: ${{ inputs.ref || github.sha }} persist-credentials: false + - name: Configure isolated Corepack path + run: echo "COREPACK_HOME=${RUNNER_TEMP}/ironclaw-corepack" >> "$GITHUB_ENV" + - name: Free disk space run: | df -h / @@ -608,6 +611,9 @@ jobs: ref: ${{ inputs.ref || github.sha }} persist-credentials: false + - name: Configure isolated Corepack path + run: echo "COREPACK_HOME=${RUNNER_TEMP}/ironclaw-corepack" >> "$GITHUB_ENV" + - name: Free disk space run: | df -h / @@ -862,6 +868,9 @@ jobs: ref: ${{ inputs.ref || github.sha }} persist-credentials: false + - name: Configure isolated Corepack path + run: echo "COREPACK_HOME=${RUNNER_TEMP}/ironclaw-corepack" >> "$GITHUB_ENV" + - name: Install mold run: | scripts/ci/install-ci-apt-packages.sh mold diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index fdce69ee0fb..b50025db917 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -22,6 +22,7 @@ run_probe() { GOOGLE_APPLICATION_CREDENTIALS="/developer/credential.json" \ LLM_BACKEND="ambient-provider" \ REBORN_TOOL_DISCLOSURE="Bridged" \ + COREPACK_HOME="${probe_dir}/corepack" \ PLAYWRIGHT_BROWSERS_PATH="${probe_dir}/playwright-browsers" \ IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ "${runner}" -- bash -c ' @@ -64,6 +65,10 @@ run_probe() { [[ "${TZ}" == "UTC" ]] [[ "${LANG}" == "C.UTF-8" ]] [[ "${LC_ALL}" == "C.UTF-8" ]] + if [[ "${COREPACK_HOME:-}" != */corepack ]]; then + echo "explicit Corepack toolchain path was not preserved" >&2 + exit 36 + fi if [[ "${PLAYWRIGHT_BROWSERS_PATH:-}" != */playwright-browsers ]]; then echo "explicit Playwright browser toolchain path was not preserved" >&2 exit 35 @@ -277,6 +282,16 @@ do fi done +corepack_path_count="$( + grep -Fc \ + 'COREPACK_HOME=${RUNNER_TEMP}/ironclaw-corepack' \ + "${repo_root}/.github/workflows/reborn-tests.yml" +)" +if [[ "${corepack_path_count}" != "3" ]]; then + echo "expected three guarded Rust lanes to pin COREPACK_HOME, found ${corepack_path_count}" >&2 + exit 1 +fi + for stage_call in \ "prepare_command_dependencies" \ "run_crate_tests" \ From 89b4ae158b32bf70064c72856d0fca116de8ba13 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 16:16:22 +0300 Subject: [PATCH 17/22] fix(ci): prefetch hermetic Postgres image --- .github/workflows/reborn-tests.yml | 15 +++++++++++++++ scripts/ci/run-hermetic-deterministic-suite.sh | 15 +++++++++++++++ scripts/ci/test-hermetic-test-process.sh | 11 +++++++++++ 3 files changed, 41 insertions(+) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index 1d7c6118801..55f71845105 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -260,6 +260,7 @@ jobs: BUCKET_PACKAGES: ${{ toJSON(matrix.bucket.packages) }} run: | needs_webui_node=false + needs_postgres_image=false sccache_dist_enabled=true while IFS= read -r package; do @@ -272,6 +273,12 @@ jobs: ;; esac + case "${package}" in + ironclaw_reborn_composition|ironclaw_triggers) + needs_postgres_image=true + ;; + esac + case "${package}" in ironclaw|ironclaw_first_party_extension_ports|ironclaw_host_runtime|ironclaw_loop_host|ironclaw_product|ironclaw_runner|ironclaw_reborn_composition|ironclaw_webui|ironclaw_wasm) sccache_dist_enabled=false @@ -280,8 +287,13 @@ jobs: done < <(printf '%s\n' "${BUCKET_PACKAGES}" | jq -r '.[]') echo "needs_webui_node=${needs_webui_node}" >> "${GITHUB_OUTPUT}" + echo "needs_postgres_image=${needs_postgres_image}" >> "${GITHUB_OUTPUT}" echo "sccache_dist_enabled=${sccache_dist_enabled}" >> "${GITHUB_OUTPUT}" + - name: Pre-pull Postgres test image + if: ${{ steps.bucket-settings.outputs.needs_postgres_image == 'true' }} + run: docker pull postgres:16-alpine + - name: Enable pnpm for setup-node cache if: ${{ steps.bucket-settings.outputs.needs_webui_node == 'true' }} run: corepack enable pnpm @@ -621,6 +633,9 @@ jobs: docker system prune -af || true df -h / + - name: Pre-pull Postgres test image + run: docker pull postgres:16-alpine + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index 695c0248d1b..04db70c4f44 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -9,6 +9,7 @@ if [[ "$#" -gt 0 ]]; then fi frontend_corepack_home="" +postgres_image_prepared=0 cleanup() { if [[ -n "${frontend_corepack_home}" && -d "${frontend_corepack_home}" ]]; then @@ -33,6 +34,18 @@ prepare_command_dependencies() { cargo fetch --locked } +prepare_postgres_test_image() { + if [[ "${postgres_image_prepared}" == "1" ]]; then + return + fi + if ! command -v docker >/dev/null 2>&1; then + echo "Docker is required to prepare postgres:16-alpine for hermetic tests" >&2 + return 2 + fi + docker pull postgres:16-alpine + postgres_image_prepared=1 +} + run_root_partitions() { local partition for partition in 0 1 2 3; do @@ -88,6 +101,7 @@ discover_reborn_packages() { run_crate_tests() { local package feature_flags + prepare_postgres_test_image while IFS= read -r package; do feature_flags="$("${repo_root}/scripts/ci/package-feature-flags.sh" "${package}")" # shellcheck disable=SC2086 # feature_flags is the checked-in CI argument list. @@ -97,6 +111,7 @@ run_crate_tests() { run_integration_tier() { local test_name + prepare_postgres_test_image while IFS= read -r test_name; do [[ "${test_name}" == --test ]] && continue run cargo test -p ironclaw_reborn_integration_tests \ diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index b50025db917..d5eed2a4951 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -292,8 +292,19 @@ if [[ "${corepack_path_count}" != "3" ]]; then exit 1 fi +postgres_pull_count="$( + grep -Fc \ + 'run: docker pull postgres:16-alpine' \ + "${repo_root}/.github/workflows/reborn-tests.yml" +)" +if [[ "${postgres_pull_count}" != "2" ]]; then + echo "expected two guarded Rust lanes to pre-pull Postgres, found ${postgres_pull_count}" >&2 + exit 1 +fi + for stage_call in \ "prepare_command_dependencies" \ + "prepare_postgres_test_image" \ "run_crate_tests" \ "run_root_partitions" \ "run_integration_tier" \ From 493210fcadff36da0839e744db00d1a2f3379d29 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 16:25:28 +0300 Subject: [PATCH 18/22] fix(ci): prepare direct WebUI crate bucket --- .github/workflows/reborn-tests.yml | 2 +- scripts/ci/test-hermetic-test-process.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index 55f71845105..5a6f55f3e7d 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -268,7 +268,7 @@ jobs: # These packages link `ironclaw_webui`, whose build script compiles # the Vite frontend and therefore needs a Node/pnpm toolchain. case "${package}" in - ironclaw|ironclaw_reborn_composition) + ironclaw|ironclaw_reborn_composition|ironclaw_webui) needs_webui_node=true ;; esac diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index d5eed2a4951..ff469055834 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -270,6 +270,7 @@ for workflow_contract in \ ".github/workflows/reborn-tests.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1" \ ".github/workflows/reborn-tests.yml:Install WebUI frontend dependencies for coverage" \ ".github/workflows/reborn-tests.yml:Install WebUI frontend dependencies for QA replay" \ + ".github/workflows/reborn-tests.yml:ironclaw|ironclaw_reborn_composition|ironclaw_webui)" \ ".github/workflows/reborn-e2e.yml:scripts/ci/run-hermetic-deterministic-suite.sh" \ ".github/workflows/reborn-e2e.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE" \ ".github/workflows/code_style.yml:scripts/ci/test-hermetic-test-process.sh" From 62b637555de06818684cbd42d3f871e322833afb Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 16:54:06 +0300 Subject: [PATCH 19/22] fix(ci): share Reborn package discovery --- .github/workflows/reborn-tests.yml | 73 +------------------ scripts/ci/discover-reborn-package-crates.sh | 69 ++++++++++++++++++ scripts/ci/reborn-local-coverage-ratchet.sh | 48 +----------- .../ci/run-hermetic-deterministic-suite.sh | 40 +--------- scripts/ci/test-hermetic-test-process.sh | 2 + 5 files changed, 74 insertions(+), 158 deletions(-) create mode 100755 scripts/ci/discover-reborn-package-crates.sh diff --git a/.github/workflows/reborn-tests.yml b/.github/workflows/reborn-tests.yml index 5a6f55f3e7d..9fd2876a6d4 100644 --- a/.github/workflows/reborn-tests.yml +++ b/.github/workflows/reborn-tests.yml @@ -134,78 +134,7 @@ jobs: - id: packages name: Build package matrix from Reborn crate families run: | - # Reborn/product crate families that are always tested, including - # crates the shipped binary does not link (channel adapters, webui_v2). - allowlist_packages="$( - cargo metadata --no-deps --format-version 1 \ - | jq -c ' - [ - .packages[] - | select( - ( - (.name == "ironclaw") - or (.name == "ironclaw_runner") - or (.name | startswith("ironclaw_reborn")) - or (.name | startswith("ironclaw_product")) - or (.name == "ironclaw_architecture") - or (.name == "ironclaw_slack_extension") - or (.name == "ironclaw_telegram_extension") - or (.name == "ironclaw_telegram_v2_adapter") - or (.name | startswith("ironclaw_webui")) - ) - # The root workspace package (test-only host for the - # reborn integration [[test]] targets) matches - # startswith("ironclaw_reborn") by NAME only. Its - # `[[test]]` suites run via the dedicated root / - # integration-coverage / group jobs — never as a - # per-crate bucket, where `--all-targets` would - # redundantly (and, for env-sensitive integration - # suites, incorrectly) re-run them. Pre-Tier-B this was - # implicit: the host package was `ironclaw_legacy`, - # which never matched the allowlist. - and (.name != "ironclaw_reborn_integration_tests") - ) - | .name - ] - | unique - ' - )" - - # The full canonical ironclaw dependency closure: every workspace - # crate the shipped Reborn binary links (normal + build deps). Running - # each crate's own suite on every PR is the "run everything on every - # PR" gate -- a green reborn-tests run means the whole closure is - # green, so the ~43 shared crates (auth, host_runtime, skills, - # extensions, ...) can no longer regress unnoticed. - closure_packages="$( - comm -12 \ - <(cargo tree -p ironclaw -e normal,build --prefix none \ - | grep -oE 'ironclaw_[a-z0-9_]+' \ - | sort -u) \ - <(cargo metadata --no-deps --format-version 1 \ - | jq -r '.packages[].name' \ - | sort -u) \ - | jq -R -s -c 'split("\n") | map(select(length > 0))' - )" - - if [ -z "${closure_packages}" ] || [ "${closure_packages}" = "[]" ]; then - echo "No Reborn CLI workspace dependency closure crates discovered" >&2 - exit 1 - fi - - # Union so closure coverage never drops the non-closure allowlist crates. - packages="$( - jq -n -c \ - --argjson allowlist "${allowlist_packages}" \ - --argjson closure "${closure_packages}" \ - '$allowlist + $closure | unique' - )" - - if [ -z "${packages}" ] || [ "${packages}" = "[]" ]; then - echo "No Reborn workspace crates discovered" >&2 - exit 1 - fi - + packages="$(scripts/ci/discover-reborn-package-crates.sh)" echo "packages=${packages}" >> "$GITHUB_OUTPUT" buckets="$(scripts/ci/reborn-crate-test-buckets.sh "${packages}")" echo "buckets=${buckets}" >> "$GITHUB_OUTPUT" diff --git a/scripts/ci/discover-reborn-package-crates.sh b/scripts/ci/discover-reborn-package-crates.sh new file mode 100755 index 00000000000..01a71a3b768 --- /dev/null +++ b/scripts/ci/discover-reborn-package-crates.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Canonical package set for the deterministic Reborn crate lanes. Keep both the +# local runner and CI matrix on this helper so package-family changes cannot +# silently diverge. +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${repo_root}" + +# Reborn/product crate families that are always tested, including crates the +# shipped binary does not link (channel adapters and WebUI packages). The +# integration-test host is excluded because its [[test]] targets run in the +# dedicated root and integration lanes; adding --all-targets here would execute +# those environment-sensitive suites twice. +allowlist="$( + cargo metadata --no-deps --format-version 1 \ + | jq -c ' + [ + .packages[] + | select( + ( + (.name == "ironclaw") + or (.name == "ironclaw_runner") + or (.name | startswith("ironclaw_reborn")) + or (.name | startswith("ironclaw_product")) + or (.name == "ironclaw_architecture") + or (.name == "ironclaw_slack_extension") + or (.name == "ironclaw_telegram_extension") + or (.name == "ironclaw_telegram_v2_adapter") + or (.name | startswith("ironclaw_webui")) + ) + and (.name != "ironclaw_reborn_integration_tests") + ) + | .name + ] + | unique + ' +)" + +# Every workspace crate linked by the shipped binary through a normal or build +# dependency. The union below keeps non-closure allowlist crates covered too. +closure="$( + comm -12 \ + <(cargo tree -p ironclaw -e normal,build --prefix none \ + | grep -oE 'ironclaw_[a-z0-9_]+' \ + | LC_ALL=C sort -u) \ + <(cargo metadata --no-deps --format-version 1 \ + | jq -r '.packages[].name' \ + | LC_ALL=C sort -u) \ + | jq -R -s -c 'split("\n") | map(select(length > 0))' +)" + +if [[ -z "${closure}" || "${closure}" == "[]" ]]; then + echo "No Reborn CLI workspace dependency closure crates discovered" >&2 + exit 1 +fi + +packages="$( + jq -n -c \ + --argjson allowlist "${allowlist}" \ + --argjson closure "${closure}" \ + '$allowlist + $closure | unique' +)" +if [[ -z "${packages}" || "${packages}" == "[]" ]]; then + echo "No Reborn workspace crates discovered" >&2 + exit 1 +fi + +printf '%s\n' "${packages}" diff --git a/scripts/ci/reborn-local-coverage-ratchet.sh b/scripts/ci/reborn-local-coverage-ratchet.sh index 2ba87094178..6e41c6fda46 100755 --- a/scripts/ci/reborn-local-coverage-ratchet.sh +++ b/scripts/ci/reborn-local-coverage-ratchet.sh @@ -55,53 +55,7 @@ echo "==> reborn coverage ratchet: preparing ${out_dir}" mkdir -p "${out_dir}/packages" "${out_dir}/buckets" "${out_dir}/lanes" find "${out_dir}/packages" "${out_dir}/buckets" "${out_dir}/lanes" -type f -name '*.lcov' -delete -allowlist_packages="$( - cargo metadata --no-deps --format-version 1 \ - | jq -c ' - [ - .packages[] - | select( - ( - (.name == "ironclaw") - or (.name == "ironclaw_runner") - or (.name | startswith("ironclaw_reborn")) - or (.name | startswith("ironclaw_product")) - or (.name == "ironclaw_architecture") - or (.name == "ironclaw_slack_extension") - or (.name == "ironclaw_telegram_extension") - or (.name == "ironclaw_telegram_v2_adapter") - or (.name | startswith("ironclaw_webui")) - ) - and (.name != "ironclaw_reborn_integration_tests") - ) - | .name - ] - | unique - ' -)" - -closure_packages="$( - comm -12 \ - <(cargo tree -p ironclaw -e normal,build --prefix none \ - | grep -oE 'ironclaw_[a-z0-9_]+' \ - | sort -u) \ - <(cargo metadata --no-deps --format-version 1 \ - | jq -r '.packages[].name' \ - | sort -u) \ - | jq -R -s -c 'split("\n") | map(select(length > 0))' -)" - -packages="$( - jq -n -c \ - --argjson allowlist "${allowlist_packages}" \ - --argjson closure "${closure_packages}" \ - '$allowlist + $closure | unique' -)" - -if [ -z "${packages}" ] || [ "${packages}" = "[]" ]; then - echo "No Reborn workspace crates discovered" >&2 - exit 1 -fi +packages="$("${script_dir}/discover-reborn-package-crates.sh")" buckets="$("${script_dir}/reborn-crate-test-buckets.sh" "${packages}")" diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index 04db70c4f44..4a803159b60 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -58,45 +58,7 @@ run_root_partitions() { } discover_reborn_packages() { - local allowlist closure - allowlist="$( - cargo metadata --no-deps --format-version 1 \ - | jq -c ' - [ - .packages[] - | select( - ( - (.name == "ironclaw") - or (.name == "ironclaw_runner") - or (.name | startswith("ironclaw_reborn")) - or (.name | startswith("ironclaw_product")) - or (.name == "ironclaw_architecture") - or (.name == "ironclaw_slack_extension") - or (.name == "ironclaw_telegram_extension") - or (.name == "ironclaw_telegram_v2_adapter") - or (.name | startswith("ironclaw_webui")) - ) - and (.name != "ironclaw_reborn_integration_tests") - ) - | .name - ] - | unique - ' - )" - closure="$( - comm -12 \ - <(cargo tree -p ironclaw -e normal,build --prefix none \ - | grep -oE 'ironclaw_[a-z0-9_]+' \ - | LC_ALL=C sort -u) \ - <(cargo metadata --no-deps --format-version 1 \ - | jq -r '.packages[].name' \ - | LC_ALL=C sort -u) \ - | jq -R -s -c 'split("\n") | map(select(length > 0))' - )" - jq -n -r \ - --argjson allowlist "${allowlist}" \ - --argjson closure "${closure}" \ - '$allowlist + $closure | unique | .[]' + "${repo_root}/scripts/ci/discover-reborn-package-crates.sh" | jq -r '.[]' } run_crate_tests() { diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index ff469055834..07476a33b19 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -265,6 +265,7 @@ fi for workflow_contract in \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh groups" \ + ".github/workflows/reborn-tests.yml:scripts/ci/discover-reborn-package-crates.sh" \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh command" \ ".github/workflows/reborn-tests.yml:scripts/ci/run-hermetic-deterministic-suite.sh prepare-command" \ ".github/workflows/reborn-tests.yml:IRONCLAW_HERMETIC_SUITE_SKIP_PREPARE=1" \ @@ -306,6 +307,7 @@ fi for stage_call in \ "prepare_command_dependencies" \ "prepare_postgres_test_image" \ + "discover-reborn-package-crates.sh" \ "run_crate_tests" \ "run_root_partitions" \ "run_integration_tier" \ From f0b3c407709954a78bd9e0f09dc12ffd650980c2 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 17:59:08 +0300 Subject: [PATCH 20/22] fix(ci): preserve explicit Emulate fixture bearer --- scripts/ci/run-hermetic-test-process.sh | 3 ++- scripts/ci/test-hermetic-test-process.sh | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run-hermetic-test-process.sh b/scripts/ci/run-hermetic-test-process.sh index 2222f1fb69b..a45bfae14c6 100755 --- a/scripts/ci/run-hermetic-test-process.sh +++ b/scripts/ci/run-hermetic-test-process.sh @@ -51,7 +51,8 @@ env_args=() if [[ "${sabotage}" != "env" ]]; then while IFS='=' read -r key _; do case "${key}" in - IRONCLAW_EMULATE_CLI|IRONCLAW_GENERATED_SEQUENCE_DEPTH|IRONCLAW_JOURNEY_ORDER) + IRONCLAW_E2E_EMULATE_SLACK_CHANNEL_BEARER|IRONCLAW_EMULATE_CLI|\ + IRONCLAW_GENERATED_SEQUENCE_DEPTH|IRONCLAW_JOURNEY_ORDER) ;; IRONCLAW_*) env_args+=("-u" "${key}") diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 07476a33b19..a68c4170faf 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -24,6 +24,7 @@ run_probe() { REBORN_TOOL_DISCLOSURE="Bridged" \ COREPACK_HOME="${probe_dir}/corepack" \ PLAYWRIGHT_BROWSERS_PATH="${probe_dir}/playwright-browsers" \ + IRONCLAW_E2E_EMULATE_SLACK_CHANNEL_BEARER="emulate-slack-channel-token" \ IRONCLAW_HERMETIC_SABOTAGE="${sabotage}" \ "${runner}" -- bash -c ' set -euo pipefail @@ -73,6 +74,10 @@ run_probe() { echo "explicit Playwright browser toolchain path was not preserved" >&2 exit 35 fi + if [[ "${IRONCLAW_E2E_EMULATE_SLACK_CHANNEL_BEARER:-}" != "emulate-slack-channel-token" ]]; then + echo "explicit Emulate Slack fixture bearer was not preserved" >&2 + exit 37 + fi if [[ "${PYTHONHASHSEED:-}" != "0" ]]; then echo "deterministic Python hash seed is not injected" >&2 exit 34 From 701f093a7f2f93a0f8c5f6e726276099d7476d04 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 18:35:50 +0300 Subject: [PATCH 21/22] fix(ci): close hermetic review gaps --- .github/workflows/reborn-e2e.yml | 2 + docs/internal/hermetic-deterministic-suite.md | 18 +++- scripts/ci/hermetic-network-guard.c | 88 ++++++++++++++++++- scripts/ci/hermetic-network-probe.c | 47 +++++++++- scripts/ci/hermetic-network-runner.sh | 23 +++-- .../ci/run-hermetic-deterministic-suite.sh | 3 + scripts/ci/run-hermetic-test-process.sh | 86 +++++++++--------- scripts/ci/test-hermetic-test-process.sh | 47 +++++++++- tests/e2e/hermetic_process.py | 2 +- 9 files changed, 261 insertions(+), 55 deletions(-) diff --git a/.github/workflows/reborn-e2e.yml b/.github/workflows/reborn-e2e.yml index 541987538c1..9cc6846d47e 100644 --- a/.github/workflows/reborn-e2e.yml +++ b/.github/workflows/reborn-e2e.yml @@ -253,6 +253,8 @@ jobs: - name: Validate product-surface evidence contracts run: >- + scripts/ci/run-hermetic-deterministic-suite.sh + command pytest tests/e2e/scenarios/test_product_surface_coverage.py tests/e2e/scenarios/test_provider_capability_inventory.py diff --git a/docs/internal/hermetic-deterministic-suite.md b/docs/internal/hermetic-deterministic-suite.md index d82c111b147..fb87f5b9ec0 100644 --- a/docs/internal/hermetic-deterministic-suite.md +++ b/docs/internal/hermetic-deterministic-suite.md @@ -30,7 +30,16 @@ only that suite-owned Corepack cache. Every command and descendant inside the suite is guarded, so remote compiler wrappers are disabled rather than treated as a network exception. -Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. That boundary: +Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. The boundary +uses a default-deny environment allowlist: only required compiler/tool paths and +named deterministic test controls survive. Cargo receives a temporary home that +links only the existing offline `registry` and `git` caches, never host Cargo +credentials or configuration; the resolved Rust sysroot is placed directly on +`PATH` without exporting the host Rustup home. Compiler output remains in the +repository's explicit `target/` build directory so prebuilt E2E binaries and +incremental CI artifacts keep their documented paths. + +That boundary: - removes real provider credentials and ambient provider/LLM behavior; - gives the process a fresh temporary home, IronClaw base/reborn homes, @@ -43,9 +52,10 @@ Every stage runs through `scripts/ci/run-hermetic-test-process.sh`. That boundar process sandbox so SIP-protected launchers remain fail-closed. On Linux, route-only UDP association is permitted because it sends no packet; explicit and connected UDP transmission through `send`, `sendto`, `sendmsg`, - `sendmmsg`, `write`, and `writev` remains blocked. This lets Chromium inspect - a route/source address without weakening the egress boundary. The coarser - macOS process sandbox may reject the harmless association as well. + `sendmmsg`, `sendfile`, `write`, and `writev` remains blocked. This lets + Chromium inspect a route/source address without weakening the egress + boundary. The coarser macOS process sandbox may reject the harmless + association as well. Rust wall-clock and random behavior is not overridden through process-global environment variables. Time-sensitive domain tests use their owning typed diff --git a/scripts/ci/hermetic-network-guard.c b/scripts/ci/hermetic-network-guard.c index 3f40e261133..f2f7e8cebe7 100644 --- a/scripts/ci/hermetic-network-guard.c +++ b/scripts/ci/hermetic-network-guard.c @@ -14,6 +14,24 @@ #include #include +#if defined(__APPLE__) +typedef int (*sendfile_fn)( + int, + int, + off_t, + off_t *, + struct sf_hdtr *, + int +); +#else +#include +typedef ssize_t (*sendfile_fn)(int, int, off_t *, size_t); +#endif + +static sendfile_fn real_sendfile(void) { + return (sendfile_fn)dlsym(RTLD_NEXT, "sendfile"); +} + #if !defined(__APPLE__) typedef int (*connect_fn)(int, const struct sockaddr *, socklen_t); typedef ssize_t (*send_fn)(int, const void *, size_t, int); @@ -80,6 +98,15 @@ static int socket_is_datagram(int socket_fd) { return status == 0 && socket_type == SOCK_DGRAM; } +static int descriptor_is_socket(int fd) { + int socket_type = 0; + socklen_t length = sizeof(socket_type); + int saved_errno = errno; + int status = getsockopt(fd, SOL_SOCKET, SO_TYPE, &socket_type, &length); + errno = saved_errno; + return status == 0; +} + static const struct sockaddr *non_loopback_destination( int socket_fd, const struct sockaddr *address, @@ -294,9 +321,59 @@ static ssize_t guarded_sendto( #endif } +#if defined(__APPLE__) +static int guarded_sendfile( + int input_fd, + int socket_fd, + off_t offset, + off_t *length, + struct sf_hdtr *headers, + int flags +) { + struct sockaddr_storage peer; + const struct sockaddr *blocked = + non_loopback_destination(socket_fd, NULL, &peer); + if (blocked != NULL) { + record_violation(blocked); + errno = EPERM; + return -1; + } + sendfile_fn function = real_sendfile(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(input_fd, socket_fd, offset, length, headers, flags); +} +#else +static ssize_t guarded_sendfile( + int socket_fd, + int input_fd, + off_t *offset, + size_t count +) { + struct sockaddr_storage peer; + const struct sockaddr *blocked = + non_loopback_destination(socket_fd, NULL, &peer); + if (blocked != NULL) { + record_violation(blocked); + errno = EPERM; + return -1; + } + sendfile_fn function = real_sendfile(); + if (function == NULL) { + errno = ENOSYS; + return -1; + } + return function(socket_fd, input_fd, offset, count); +} +#endif + static ssize_t guarded_write(int fd, const void *buffer, size_t length) { struct sockaddr_storage peer; - const struct sockaddr *blocked = non_loopback_destination(fd, NULL, &peer); + const struct sockaddr *blocked = descriptor_is_socket(fd) + ? non_loopback_destination(fd, NULL, &peer) + : NULL; if (blocked != NULL) { record_violation(blocked); errno = EPERM; @@ -320,7 +397,9 @@ static ssize_t guarded_writev( int iovec_count ) { struct sockaddr_storage peer; - const struct sockaddr *blocked = non_loopback_destination(fd, NULL, &peer); + const struct sockaddr *blocked = descriptor_is_socket(fd) + ? non_loopback_destination(fd, NULL, &peer) + : NULL; if (blocked != NULL) { record_violation(blocked); errno = EPERM; @@ -352,6 +431,7 @@ DYLD_INTERPOSE(guarded_connect, connect); DYLD_INTERPOSE(guarded_send, send); DYLD_INTERPOSE(guarded_sendmsg, sendmsg); DYLD_INTERPOSE(guarded_sendto, sendto); +DYLD_INTERPOSE(guarded_sendfile, sendfile); DYLD_INTERPOSE(guarded_write, write); DYLD_INTERPOSE(guarded_writev, writev); #else @@ -387,6 +467,10 @@ ssize_t sendto( return guarded_sendto(socket_fd, buffer, length, flags, address, address_length); } +ssize_t sendfile(int socket_fd, int input_fd, off_t *offset, size_t count) { + return guarded_sendfile(socket_fd, input_fd, offset, count); +} + ssize_t write(int fd, const void *buffer, size_t length) { return guarded_write(fd, buffer, length); } diff --git a/scripts/ci/hermetic-network-probe.c b/scripts/ci/hermetic-network-probe.c index 0faad08228c..d7158fd8887 100644 --- a/scripts/ci/hermetic-network-probe.c +++ b/scripts/ci/hermetic-network-probe.c @@ -3,13 +3,19 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#if defined(__linux__) +#include +#endif + int main(int argc, char **argv) { if (argc != 2 && argc != 3) { return 2; @@ -19,7 +25,8 @@ int main(int argc, char **argv) { int use_udp = strcmp(mode, "udp") == 0 || strcmp(mode, "udp-connected") == 0 - || strcmp(mode, "udp-connect-only") == 0; + || strcmp(mode, "udp-connect-only") == 0 + || strcmp(mode, "udp-sendfile") == 0; #if defined(__linux__) use_udp = use_udp || strcmp(mode, "udp-sendmmsg") == 0; #endif @@ -76,6 +83,44 @@ int main(int argc, char **argv) { if (network_status == 0 && strcmp(mode, "udp-connected") == 0) { const char byte = 'x'; network_status = (int)send(fd, &byte, sizeof(byte), 0); + } else if (network_status == 0 && strcmp(mode, "udp-sendfile") == 0) { + const char *temporary_root = getenv("TMPDIR"); + if (temporary_root == NULL || temporary_root[0] == '\0') { + close(fd); + return 6; + } + char input_path[PATH_MAX]; + int path_length = snprintf( + input_path, + sizeof(input_path), + "%s/ironclaw-sendfile-probe.XXXXXX", + temporary_root + ); + if (path_length < 0 || (size_t)path_length >= sizeof(input_path)) { + close(fd); + return 6; + } + int input_fd = mkstemp(input_path); + if (input_fd < 0) { + close(fd); + return 7; + } + const char byte = 'x'; + if (write(input_fd, &byte, sizeof(byte)) != (ssize_t)sizeof(byte)) { + close(input_fd); + unlink(input_path); + close(fd); + return 8; + } +#if defined(__APPLE__) + off_t length = sizeof(byte); + network_status = sendfile(input_fd, fd, 0, &length, NULL, 0); +#else + off_t offset = 0; + network_status = (int)sendfile(fd, input_fd, &offset, sizeof(byte)); +#endif + close(input_fd); + unlink(input_path); } else if (network_status == 0 && strcmp(mode, "udp-write") == 0) { const char byte = 'x'; network_status = (int)write(fd, &byte, sizeof(byte)); diff --git a/scripts/ci/hermetic-network-runner.sh b/scripts/ci/hermetic-network-runner.sh index 381b12c4fb8..b05790d24fb 100755 --- a/scripts/ci/hermetic-network-runner.sh +++ b/scripts/ci/hermetic-network-runner.sh @@ -26,12 +26,23 @@ case "$(uname -s)" in # SIP-protected Apple executables strip DYLD_* before their descendants # inherit it. The process sandbox keeps the whole tree fail-closed; the # interposer still records actionable diagnostics for ordinary binaries. - guarded_command=( - sandbox-exec - -p - '(version 1) (allow default) (deny network-outbound) (allow network-outbound (remote ip "localhost:*"))' - "$@" - ) + if [[ "${IRONCLAW_HERMETIC_SANDBOX_ACTIVE:-0}" != "1" ]]; then + if ! command -v sandbox-exec >/dev/null 2>&1; then + echo "sandbox-exec is required for fail-closed macOS hermetic networking" >&2 + exit 2 + fi + if ! sandbox-exec -p '(version 1) (allow default)' /usr/bin/true; then + echo "sandbox-exec is present but cannot enforce a process sandbox" >&2 + exit 2 + fi + export IRONCLAW_HERMETIC_SANDBOX_ACTIVE=1 + guarded_command=( + sandbox-exec + -p + '(version 1) (allow default) (deny network-outbound) (allow network-outbound (remote ip "localhost:*"))' + "$@" + ) + fi ;; *) echo "unsupported platform for hermetic network guard: $(uname -s)" >&2 diff --git a/scripts/ci/run-hermetic-deterministic-suite.sh b/scripts/ci/run-hermetic-deterministic-suite.sh index 4a803159b60..38e31dede42 100755 --- a/scripts/ci/run-hermetic-deterministic-suite.sh +++ b/scripts/ci/run-hermetic-deterministic-suite.sh @@ -89,8 +89,10 @@ run_python_e2e() { fi run pytest \ tests/e2e/scenarios/test_reborn_webui_v2_smoke.py \ + tests/e2e/scenarios/test_reborn_webui_v2_sso.py \ -v --timeout=120 run pytest \ + tests/e2e/scenarios/test_product_surface_coverage.py \ tests/e2e/scenarios/test_provider_capability_inventory.py \ tests/e2e/scenarios/test_journey_coverage.py \ tests/e2e/scenarios/test_emulate_reborn_provider_contracts.py \ @@ -177,6 +179,7 @@ case "${stage}" in run_frontend_tests ;; python-e2e) + prepare_rust_dependencies run_python_e2e ;; prepare-command) diff --git a/scripts/ci/run-hermetic-test-process.sh b/scripts/ci/run-hermetic-test-process.sh index a45bfae14c6..0482210031b 100755 --- a/scripts/ci/run-hermetic-test-process.sh +++ b/scripts/ci/run-hermetic-test-process.sh @@ -49,43 +49,52 @@ fi env_args=() if [[ "${sabotage}" != "env" ]]; then - while IFS='=' read -r key _; do + while IFS= read -r key; do case "${key}" in + AR|CC|CXX|LD|RANLIB|STRIP|PKG_CONFIG|PKG_CONFIG_PATH|\ + OPENSSL_DIR|OPENSSL_INCLUDE_DIR|OPENSSL_LIB_DIR|LIBCLANG_PATH|\ + SDKROOT|MACOSX_DEPLOYMENT_TARGET|DEVELOPER_DIR|\ + LD_LIBRARY_PATH|DYLD_LIBRARY_PATH|LIBRARY_PATH|CPATH|\ + CI|GITHUB_ACTIONS|TERM|COLORTERM|NO_COLOR|FORCE_COLOR|\ + CARGO_INCREMENTAL|CARGO_PROFILE_DEV_DEBUG|CARGO_PROFILE_TEST_DEBUG|CARGO_TEST_ARGS|\ + RUSTFLAGS|RUST_MIN_STACK|COREPACK_HOME|PLAYWRIGHT_BROWSERS_PATH|\ + PROPTEST_CASES|REBORN_COV_LANE_INDEX|REBORN_COV_LANE_MODE|\ + REBORN_COV_LANE_PARTITIONS|REBORN_COV_LANE_TEST_TIMEOUT|\ + REBORN_GROUP_TEST_TIMEOUT|REBORN_ROOT_TEST_PARTITION|\ + REBORN_ROOT_TEST_PARTITIONS|REBORN_ROOT_TEST_TIMEOUT|\ IRONCLAW_E2E_EMULATE_SLACK_CHANNEL_BEARER|IRONCLAW_EMULATE_CLI|\ IRONCLAW_GENERATED_SEQUENCE_DEPTH|IRONCLAW_JOURNEY_ORDER) ;; - IRONCLAW_*) - env_args+=("-u" "${key}") - ;; - *_API_KEY|*_TOKEN|*_SECRET|*_CREDENTIALS|*_PASSWORD|*_PRIVATE_KEY) - env_args+=("-u" "${key}") - ;; - ANTHROPIC_*|OPENAI_*|NEARAI_*|GOOGLE_*|GITHUB_TOKEN|GH_TOKEN|SLACK_*|TELEGRAM_*|NOTION_*|EXA_*|BRAVE_*|AWS_*|AZURE_*|COHERE_*|MISTRAL_*|GROQ_*|\ - LLM_*|OLLAMA_*|REBORN_TOOL_DISCLOSURE|DATABASE_URL|LIBSQL_PATH|SECRETS_MASTER_KEY|\ - HTTP_PROXY|HTTPS_PROXY|ALL_PROXY|NO_PROXY|http_proxy|https_proxy|all_proxy|no_proxy) + *) env_args+=("-u" "${key}") ;; esac - done <<<"$(env)" + done < <(compgen -e) fi original_home="${HOME:-}" -if [[ -n "${CARGO_HOME:-}" ]]; then - cargo_home="${CARGO_HOME}" -elif [[ -n "${original_home}" ]]; then - cargo_home="${original_home}/.cargo" -else - cargo_home="" -fi -if [[ -n "${RUSTUP_HOME:-}" ]]; then - rustup_home="${RUSTUP_HOME}" -elif [[ -n "${original_home}" ]]; then - rustup_home="${original_home}/.rustup" -else - rustup_home="" +original_cargo_home="${CARGO_HOME:-${original_home:+${original_home}/.cargo}}" +sanitized_cargo_home="${hermetic_root}/cargo-home" +mkdir -p "${sanitized_cargo_home}" +for cargo_cache in registry git; do + if [[ -n "${original_cargo_home}" && -d "${original_cargo_home}/${cargo_cache}" ]]; then + ln -s "${original_cargo_home}/${cargo_cache}" "${sanitized_cargo_home}/${cargo_cache}" + fi +done + +tool_path="${PATH:-/usr/bin:/bin}" +rust_sysroot="" +if command -v rustc >/dev/null 2>&1; then + rust_sysroot="$(rustc --print sysroot)" + if [[ -z "${rust_sysroot}" || ! -d "${rust_sysroot}/bin" ]]; then + echo "unable to resolve the installed Rust toolchain for hermetic tests" >&2 + exit 1 + fi + tool_path="${rust_sysroot}/bin:${tool_path}" fi env_args+=( + "PATH=${tool_path}" "IRONCLAW_HERMETIC_ROOT=${hermetic_root}" "IRONCLAW_DISABLE_OS_KEYCHAIN=1" "LLM_MAX_RETRIES=0" @@ -96,6 +105,8 @@ env_args+=( "TZ=UTC" "LANG=C.UTF-8" "LC_ALL=C.UTF-8" + "CARGO_HOME=${sanitized_cargo_home}" + "CARGO_TARGET_DIR=${repo_root}/target" ) if [[ "${sabotage}" != "temp" ]]; then @@ -109,14 +120,9 @@ if [[ "${sabotage}" != "temp" ]]; then "XDG_CONFIG_HOME=${hermetic_root}/xdg-config" "XDG_DATA_HOME=${hermetic_root}/xdg-data" ) +else + env_args+=("HOME=${original_home:-/}") fi -if [[ -n "${cargo_home}" ]]; then - env_args+=("CARGO_HOME=${cargo_home}") -fi -if [[ -n "${rustup_home}" ]]; then - env_args+=("RUSTUP_HOME=${rustup_home}") -fi - if [[ "${sabotage}" != "python-seed" ]]; then env_args+=( "PYTHONHASHSEED=0" @@ -124,16 +130,16 @@ if [[ "${sabotage}" != "python-seed" ]]; then fi if [[ "${sabotage}" != "network" ]]; then - host_triple="$(rustc -vV | sed -n 's/^host: //p')" - if [[ -z "${host_triple}" ]]; then - echo "unable to determine Rust host triple for the hermetic test runner" >&2 - exit 1 + env_args+=("IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY=${guard_library}") + if [[ -n "${rust_sysroot}" ]]; then + host_triple="$("${rust_sysroot}/bin/rustc" -vV | sed -n 's/^host: //p')" + if [[ -z "${host_triple}" ]]; then + echo "unable to determine Rust host triple for the hermetic test runner" >&2 + exit 1 + fi + cargo_runner_key="CARGO_TARGET_$(tr '[:lower:]-' '[:upper:]_' <<<"${host_triple}")_RUNNER" + env_args+=("${cargo_runner_key}=${network_runner}") fi - cargo_runner_key="CARGO_TARGET_$(tr '[:lower:]-' '[:upper:]_' <<<"${host_triple}")_RUNNER" - env_args+=( - "IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY=${guard_library}" - "${cargo_runner_key}=${network_runner}" - ) fi command_prefix=() diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index a68c4170faf..2201d7081b8 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -20,6 +20,14 @@ run_probe() { ANTHROPIC_API_KEY="must-not-leak" \ GITHUB_TOKEN="must-not-leak" \ GOOGLE_APPLICATION_CREDENTIALS="/developer/credential.json" \ + PERPLEXITY_KEY="must-not-leak" \ + DEEPSEEK_APIKEY="must-not-leak" \ + HF_TOKEN_FILE="/developer/hf-token" \ + SSH_AUTH_SOCK="/developer/ssh-agent.sock" \ + GPG_AGENT_INFO="/developer/gpg-agent" \ + BASH_ENV="/developer/shell-startup" \ + CARGO_TARGET_DIR="/developer/target" \ + AMBIENT_MULTILINE=$'first line\nPATH=/developer/injected' \ LLM_BACKEND="ambient-provider" \ REBORN_TOOL_DISCLOSURE="Bridged" \ COREPACK_HOME="${probe_dir}/corepack" \ @@ -32,6 +40,13 @@ run_probe() { ANTHROPIC_API_KEY \ GITHUB_TOKEN \ GOOGLE_APPLICATION_CREDENTIALS \ + PERPLEXITY_KEY \ + DEEPSEEK_APIKEY \ + HF_TOKEN_FILE \ + SSH_AUTH_SOCK \ + GPG_AGENT_INFO \ + BASH_ENV \ + AMBIENT_MULTILINE \ LLM_BACKEND \ REBORN_TOOL_DISCLOSURE do @@ -41,6 +56,22 @@ run_probe() { fi done + case "${CARGO_HOME}" in + "${IRONCLAW_HERMETIC_ROOT}"/*) ;; + *) + echo "CARGO_HOME is outside the hermetic root: ${CARGO_HOME}" >&2 + exit 38 + ;; + esac + if [[ -n "${RUSTUP_HOME+x}" ]]; then + echo "ambient RUSTUP_HOME leaked into the hermetic process" >&2 + exit 39 + fi + if [[ "${CARGO_TARGET_DIR}" != */target || "${CARGO_TARGET_DIR}" == "/developer/target" ]]; then + echo "ambient CARGO_TARGET_DIR leaked into the hermetic process" >&2 + exit 40 + fi + case "${HOME}" in "${IRONCLAW_HERMETIC_ROOT}"/*) ;; *) @@ -185,7 +216,7 @@ else exit 1 fi fi -connected_udp_modes=(udp-connected udp-write udp-writev) +connected_udp_modes=(udp-connected udp-sendfile udp-write udp-writev) if [[ "$(uname -s)" == "Linux" ]]; then connected_udp_modes+=(udp-sendmmsg) fi @@ -232,11 +263,13 @@ source = { "IRONCLAW_HERMETIC_NETWORK_VIOLATIONS": "/tmp/violations", "LD_PRELOAD": "/tmp/existing.so", "DYLD_INSERT_LIBRARIES": "/tmp/existing.dylib", + "DYLD_FORCE_FLAT_NAMESPACE": "0", "ANTHROPIC_API_KEY": "must-not-forward", } child = {} forward_hermetic_process_env(child, source) assert child["IRONCLAW_HERMETIC_NETWORK_VIOLATIONS"] == "/tmp/violations" +assert "ANTHROPIC_API_KEY" not in child if sys.platform == "darwin": assert child["DYLD_INSERT_LIBRARIES"] == "/tmp/guard.so:/tmp/existing.dylib" assert child["LD_PRELOAD"] == "/tmp/existing.so" @@ -289,6 +322,16 @@ do fi done +evidence_contract_step="$( + sed -n \ + '/- name: Validate product-surface evidence contracts/,/- id: product_surface_coverage/p' \ + "${repo_root}/.github/workflows/reborn-e2e.yml" +)" +if [[ "${evidence_contract_step}" != *"run-hermetic-deterministic-suite.sh"* ]]; then + echo "product-surface evidence contracts bypass the hermetic process boundary" >&2 + exit 1 +fi + corepack_path_count="$( grep -Fc \ 'COREPACK_HOME=${RUNNER_TEMP}/ironclaw-corepack' \ @@ -320,6 +363,8 @@ for stage_call in \ "run-reborn-group-tests.sh" \ "check-reborn-qa-fixtures.sh" \ "reborn-e2e-rust.sh" \ + "test_product_surface_coverage.py" \ + "test_reborn_webui_v2_sso.py" \ "run_python_e2e" do if ! grep -Fq "${stage_call}" "${repo_root}/scripts/ci/run-hermetic-deterministic-suite.sh"; then diff --git a/tests/e2e/hermetic_process.py b/tests/e2e/hermetic_process.py index dd6f7490f69..2795ffd543c 100644 --- a/tests/e2e/hermetic_process.py +++ b/tests/e2e/hermetic_process.py @@ -31,7 +31,7 @@ def forward_hermetic_process_env( return if sys.platform == "darwin": _prepend_preload(env, "DYLD_INSERT_LIBRARIES", guard_library) - env.setdefault("DYLD_FORCE_FLAT_NAMESPACE", "1") + env["DYLD_FORCE_FLAT_NAMESPACE"] = "1" elif sys.platform.startswith("linux"): _prepend_preload(env, "LD_PRELOAD", guard_library) From a82720d26b5e56633b42935fd6746975b5889464 Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Thu, 30 Jul 2026 19:04:34 +0300 Subject: [PATCH 22/22] fix(e2e): reject empty hermetic guard path --- scripts/ci/test-hermetic-test-process.sh | 10 ++++++++++ tests/e2e/hermetic_process.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/scripts/ci/test-hermetic-test-process.sh b/scripts/ci/test-hermetic-test-process.sh index 2201d7081b8..11373fb5a81 100755 --- a/scripts/ci/test-hermetic-test-process.sh +++ b/scripts/ci/test-hermetic-test-process.sh @@ -277,6 +277,16 @@ if sys.platform == "darwin": elif sys.platform.startswith("linux"): assert child["LD_PRELOAD"] == "/tmp/guard.so:/tmp/existing.so" assert child["DYLD_INSERT_LIBRARIES"] == "/tmp/existing.dylib" + +try: + forward_hermetic_process_env( + {}, + {"IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY": " "}, + ) +except ValueError as error: + assert "must be non-empty" in str(error) +else: + raise AssertionError("empty network guard library did not fail closed") PY set +e diff --git a/tests/e2e/hermetic_process.py b/tests/e2e/hermetic_process.py index 2795ffd543c..0804fb1eb3e 100644 --- a/tests/e2e/hermetic_process.py +++ b/tests/e2e/hermetic_process.py @@ -29,6 +29,8 @@ def forward_hermetic_process_env( guard_library = source_env.get("IRONCLAW_HERMETIC_NETWORK_GUARD_LIBRARY") if guard_library is None: return + if not guard_library.strip(): + raise ValueError("Hermetic network guard library must be non-empty") if sys.platform == "darwin": _prepend_preload(env, "DYLD_INSERT_LIBRARIES", guard_library) env["DYLD_FORCE_FLAT_NAMESPACE"] = "1"