Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions tests/reborn_integration_greeting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@
//! `ironclaw_llm` decorator chain (hermetic passthrough) → scripted `TraceLlm`
//! → assistant reply finalized in thread history. InMemory storage, no services,
//! no keys, no Docker, no `integration` feature.
//!
//! Asserts BOTH facets of the one default turn: the finalized reply (output
//! seam) and the model-visible system prompt (input seam, T0-SYSPROMPT). The
//! system-prompt assertion rides this smoke test rather than a redundant file —
//! it exercises the same `build → submit_turn` path, so consolidating avoids a
//! second full support-tree compile for zero new path coverage.

// The support tree is large and shared; a single-test file exercises only a
// slice of it, so suppress dead-code warnings on the includes (matches
Expand Down Expand Up @@ -33,4 +39,20 @@ async fn replies_to_greeting() {
.assert_reply_contains("Hello! How can I help?")
.await
.expect("reply finalized in thread history");
// Input seam (T0-SYSPROMPT): the composed capability policy is rendered
// into a `System`-role message the model actually saw this turn.
harness
.assert_system_prompt_contains("Use only visible capabilities.")
.await
.expect("composed capability policy reached the model as a system prompt");
// Negative guard: the user's own turn text appears in the captured request
// but only in a `User`-role message, so the `System`-only filter must not
// match it — proves the assertion discriminates on role, not mere presence.
assert!(
harness
.assert_system_prompt_contains("hi there")
.await
.is_err(),
"system-prompt assertion must not match user-role text"
);
}
103 changes: 103 additions & 0 deletions tests/reborn_integration_secret_injection.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
//! Reborn integration-test tier — T0-SECRET-INJECT.
//!
//! Proves credential/secret injection reaches the wire: a scripted `github.*`
//! tool call executes the real first-party GitHub WASM capability behind a
//! `GithubHarnessAuthorizer` that attaches an `InjectCredentialAccountOnce`
//! obligation. The host egress pipeline resolves the synthetic access token
//! (`ghp_fake_fixture_token`, from the harness `StaticSecretStore`) and injects
//! it as `Authorization: Bearer <token>` onto the outbound request before the
//! recording network egress captures it. The assertion reads that captured
//! request and confirms the injected credential is present on the header.
//!
//! Note on the egress lane: this harness's runtime egress recorder
//! (`runtime_http_requests()`) is inert — `try_with_host_http_egress` overwrites
//! the runtime port with the host pipeline over the recording *network* egress —
//! so injection is observable on the network lane. See
//! `assert_network_egress_header_contains` for the full mechanism.
//!
//! Security: the token is a synthetic test fixture, never a real credential.

// The support tree is large and shared; a single-test file exercises only a
// slice of it, so suppress dead-code warnings on the includes (matches
// `reborn_integration_greeting.rs`).
#[allow(dead_code)]
#[path = "support/reborn/mod.rs"]
mod reborn_support;
#[allow(dead_code)]
mod support;

use reborn_support::builder::RebornIntegrationHarness;
use reborn_support::reply::RebornScriptedReply;
use serde_json::json;

#[tokio::test]
async fn injects_credential_onto_github_egress() {
let harness = RebornIntegrationHarness::test_default()
.with_github_issue_tools()
.script([
RebornScriptedReply::tool_call(
"github.get_repo",
json!({"owner": "nearai", "repo": "ironclaw"}),
),
RebornScriptedReply::text("done"),
])
.build()
.await
.expect("harness builds");
harness
.submit_turn("fetch the ironclaw repo")
.await
.expect("turn completes");
harness
.assert_reply_contains("done")
.await
.expect("reply finalized in thread history");
// The synthetic access token was injected onto the outbound request as a
// Bearer credential by the host egress pipeline — proving injection reaches
// the wire, not just the authorizer's obligation.
harness
.assert_network_egress_header_contains(
"api.github.com/repos/nearai/ironclaw",
"authorization",
"Bearer ghp_fake_fixture_token",
)
.await
.expect("injected credential present on github egress request");

// Negative-path coverage on the SAME captured request: a regression that
// ignored the url/header-name/value inputs must not let this assertion
// pass vacuously (review comment on PR #5483).
let wrong_url = harness
.assert_network_egress_header_contains(
"api.github.com/repos/nonexistent/repo",
"authorization",
"Bearer ghp_fake_fixture_token",
)
.await
.expect_err("no captured request should match an unrelated url");
assert!(
wrong_url
.to_string()
.contains("no captured network egress request matching url")
);

let wrong_header_name = harness
.assert_network_egress_header_contains(
"api.github.com/repos/nearai/ironclaw",
"x-not-a-real-header",
"Bearer ghp_fake_fixture_token",
)
.await
.expect_err("matching url has no such header name");
assert!(wrong_header_name.to_string().contains("has header"));

let wrong_value = harness
.assert_network_egress_header_contains(
"api.github.com/repos/nearai/ironclaw",
"authorization",
"Bearer wrong-token",
)
.await
.expect_err("matching url/header present but value doesn't match");
assert!(wrong_value.to_string().contains("has header"));
}
34 changes: 28 additions & 6 deletions tests/support/reborn/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ conversation; `submit_turn`/`assert_reply_contains` take just the text.
MCP assertion (`assert_mcp_tool_called`), approval methods
(`submit_turn_until_blocked` / `approve_gate` / `deny_gate` / `enable_auto_approve`),
and the `pub(super)` capture accessors (`captured_egress_requests` /
`captured_capability_results`) the assertion file reads.
`captured_capability_results` / `captured_system_prompts`) the assertion file reads.
- `harness_mcp.rs` — the mock-MCP scaffolding extracted from `harness.rs`:
`LoopbackMcpRuntimeHttpEgress` (the real-HTTP loopback egress), the
`LoopbackMcpRuntime` type alias + `build_loopback_mcp_runtime` factory,
Expand All @@ -101,7 +101,10 @@ conversation; `submit_turn`/`assert_reply_contains` take just the text.
- `assertions.rs` — the richer egress + tool-result assertions
(`assert_egress_count` / `assert_egress_url_order` / `assert_egress_method_order`
/ `assert_egress_body_contains` / `assert_tool_result_contains` /
`assert_tool_error`).
`assert_tool_error` / `assert_network_egress_header_contains`), plus the
model-prompt assertion `assert_system_prompt_contains` (reads the scripted
`TraceLlm`'s captured requests via `captured_system_prompts`, not the egress
log).
- Tests live as flat `tests/reborn_*.rs` (Cargo requires top-level test files).

Module paths: each `tests/reborn_*.rs` declares both `#[path = "support/reborn/mod.rs"] mod reborn_support;` and `mod support;`, then `use reborn_support::builder::RebornIntegrationHarness;` / `use reborn_support::reply::RebornScriptedReply;`. Inside the support tree, siblings reference each other via `super::` and `trace_llm` via `crate::support::trace_llm` (there is no `crate::support::reborn` path). Copy the includes from `tests/reborn_integration_greeting.rs`.
Expand Down Expand Up @@ -151,6 +154,7 @@ Richer assertions in `assertions.rs` (all check the `[baseline..]` delta per thr
- `assert_egress_body_contains(url_substr, body_substr)` — body of the captured egress request whose URL contains the substring.
- `assert_tool_result_contains(needle)` — a recorded capability result's output contains the text (proves the scripted body surfaced back to the model on the *Completed* path; reads the in-process recorder).
- `assert_tool_error(class, reason)` — a persisted `ToolResultReference` envelope's parsed `safe_summary` field is of outcome `class` (`ToolErrorClass::{Failed, Denied}`) and carries `reason`. Distinct from `assert_tool_result_contains`: this reads the *Failed*/*Denied* capability-error path (persisted via `append_tool_result_reference`), not the in-process recorder, so it's the assertion for `egress_error`-scripted responses and other capability failures/denials. `class` is a typed arg (not a needle prefix) so it discriminates Failed-vs-Denied structurally — a `Failed{PolicyDenied}` and a `Denied{policy_denied}` render the same `reason` token but different classes. Parses the `safe_summary` field (not a raw-JSON substring). Scans full thread history (not baseline-sliced) — safe only for single-turn harnesses today; a multi-turn/group reuse must add baseline scoping first.
- `assert_network_egress_header_contains(url_substr, header_name, value_substr)` — reads the **network** egress lane (`captured_network_requests()`), not the runtime lane the four assertions above read. Needed for `.with_github_issue_tools()`: that harness's `try_with_host_http_egress` overwrites the runtime port with the host egress pipeline over the network recorder, so the runtime-lane `assert_egress_*` family is inert for it — assert here instead.

### Keyed HTTP responses

Expand Down Expand Up @@ -197,6 +201,22 @@ Script with `RebornScriptedReply::tool_call("mock-mcp.search", json!({}))`.

- `assert_mcp_tool_called(tool_name)` — maps `tool_name` → `"mock-mcp.<tool_name>"` and delegates to `assert_tool_invoked`.

### Credential injection (GitHub)

`.with_github_issue_tools()` wires the real GitHub first-party WASM
capabilities behind a `GithubHarnessAuthorizer`, which authorizes every
dispatch with an `InjectCredentialAccountOnce` obligation. A scripted
`github.*` tool call executes the real WASM module; its outbound HTTP
request gets a synthetic `Authorization: Bearer <token>` credential
injected by the host egress pipeline before it reaches the recording
network egress. This is the credential-injection-reaches-the-wire proof
(T0-SECRET-INJECT).

Script with `RebornScriptedReply::tool_call("github.get_repo", json!({"owner": ..., "repo": ...}))`
followed by a trailing `RebornScriptedReply::text(..)` turn.

- `assert_network_egress_header_contains(url_substr, header_name, value_substr)` — see the "Richer assertions" list below; this is the assertion for this capability.

### OAuth / product-auth

Available from crate `ironclaw_reborn_composition::test_support`, gated on
Expand Down Expand Up @@ -364,7 +384,9 @@ pub async fn run(g: &RebornIntegrationGroup) -> HarnessResult<()> {
### Per-thread baseline (R2)

Each `RebornIntegrationHarness` records `baseline_invocation_count`,
`baseline_egress_count`, and `baseline_result_count` at construction from the
shared recorder's current lengths. All assertion methods (`assert_tool_invoked`,
`assert_egress_request_matching`, etc.) slice `[baseline..]` so a thread never
spuriously passes on a prior thread's entries.
`baseline_egress_count`, `baseline_result_count`, `baseline_process_count`, and
`baseline_network_count` at construction from the shared recorder's current
lengths. All assertion methods (`assert_tool_invoked`,
`assert_egress_request_matching`, `assert_network_egress_header_contains`,
etc.) slice `[baseline..]` so a thread never spuriously passes on a prior
thread's entries.
109 changes: 103 additions & 6 deletions tests/support/reborn/assertions.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Egress + tool-result assertions for [`RebornIntegrationHarness`] — the
//! canonical, richer egress-assertion API (design §3.3 `assertions.rs`, §3.6
//! P1 ergonomics).
//! Egress + tool-result + model-prompt assertions for [`RebornIntegrationHarness`]
//! — the canonical, richer egress-assertion API (design §3.3 `assertions.rs`,
//! §3.6 P1 ergonomics).
//!
//! Slice 2 co-located three asserts in `builder.rs`
//! (`assert_reply_contains`/`assert_tool_invoked`/`assert_egress_request_matching`,
Expand All @@ -11,9 +11,18 @@
//! `pub(super)` accessors on the harness (`captured_egress_requests` /
//! `captured_capability_results`) rather than re-reaching internals.
//!
//! All of these assert over the SAME captured `RecordingRuntimeHttpEgress`
//! request log slice 2 wired — there is one egress-assertion API, not a parallel
//! one (the O-egress MCP/OAuth interceptor folds its per-URL needs in here).
//! The egress-assertion group (`assert_egress_count` / `assert_egress_url_order`
//! / `assert_egress_method_order` / `assert_egress_body_contains`) all assert
//! over the SAME captured `RecordingRuntimeHttpEgress` request log slice 2 wired
//! — there is one runtime-lane egress-assertion API, not a parallel one (the
//! O-egress MCP/OAuth interceptor folds its per-URL needs in here). The one
//! exception is `assert_network_egress_header_contains`, which reads the
//! recording *network* egress lane — required for the T0-SECRET-INJECT
//! credential-injection proof, whose harness routes through the host egress
//! pipeline over the network recorder (see that method's docs for why).
//! `assert_system_prompt_contains` reads a different capture source — the
//! scripted `TraceLlm`'s captured requests, via the harness's
//! `captured_system_prompts` accessor.

// Shared integration-test support: not every binary that mounts the
// `reborn_support` tree consumes this module (e.g. `support_unit_tests.rs`), so
Expand Down Expand Up @@ -137,6 +146,30 @@ impl RebornIntegrationHarness {
.into())
}

/// Assert some model-visible `System`-role prompt captured across all
/// requests captured by the harness so far contains `text`. Reads the
/// scripted `TraceLlm` retained before the `dyn LlmProvider` upcast —
/// proves prompt-injected content (safety banners, skill instructions,
/// profile lines) actually reached the model.
pub async fn assert_system_prompt_contains(&self, text: &str) -> HarnessResult<()> {
let prompts = self.captured_system_prompts();
if prompts.iter().any(|prompt| prompt.contains(text)) {
return Ok(());
}
let seen: Vec<String> = prompts
.iter()
.map(|prompt| match prompt.char_indices().nth(200) {
Some((cutoff, _)) => format!("{}...[truncated]", &prompt[..cutoff]),
None => prompt.clone(),
})
.collect();
Err(format!(
"no captured system prompt containing {text:?}; saw {} system message(s): {seen:?}",
prompts.len()
)
.into())
}

/// Assert a model-visible tool error of `class` carrying `reason` was
/// persisted for this thread. Unlike [`assert_tool_result_contains`] (which
/// reads the in-process recorder, populated only on the *Completed* write
Expand Down Expand Up @@ -198,6 +231,70 @@ impl RebornIntegrationHarness {
.into())
}

/// Assert that any captured **network** egress request whose URL
/// contains `url_substr` carried a header named `header_name`
/// (case-insensitive) whose value contains `value_substr`. This is the
/// credential-injection-on-the-wire proof for T0-SECRET-INJECT: a
/// host-injected `Authorization: Bearer <token>` lands on the outbound
/// request only after the egress pipeline's `apply_credential_injections`
/// step, which the recording network egress captures.
///
/// **Why the network lane, not the runtime lane:** the GitHub WASM harness
/// (`with_github_issue_tools`) wires its recording `RuntimeHttpEgress` and
/// then calls `try_with_host_http_egress`, which overwrites the runtime port
/// with the host egress pipeline over the recording *network* egress. So the
/// injected request flows through the network recorder, and the runtime-lane
/// `assert_egress_*` family (which reads `runtime_http_requests()`) is inert
/// for this wiring. Assert here instead.
///
/// Checks only the `[baseline_network_count..]` delta so a group thread never
/// spuriously matches a prior thread's request (R2), mirroring the runtime-lane
/// `assert_egress_*` family's baseline discipline even though no group
/// constructor wires `GithubIssueTools` today.
pub async fn assert_network_egress_header_contains(
&self,
url_substr: &str,
header_name: &str,
value_substr: &str,
) -> HarnessResult<()> {
let requests = self.captured_network_requests();
let mut matching = requests
.iter()
.filter(|r| r.url.contains(url_substr))
.peekable();
if matching.peek().is_none() {
let seen: Vec<&str> = requests.iter().map(|r| r.url.as_str()).collect();
return Err(format!(
"no captured network egress request matching url {url_substr:?}; saw {seen:?}"
)
.into());
}
let mut first_seen: Option<Vec<&str>> = None;
for request in matching {
if request.headers.iter().any(|(name, value)| {
name.eq_ignore_ascii_case(header_name) && value.contains(value_substr)
}) {
return Ok(());
}
if first_seen.is_none() {
first_seen = Some(
request
.headers
.iter()
.map(|(name, _)| name.as_str())
.collect(),
);
}
}
let seen = first_seen.unwrap_or_default();
Err(format!(
"no network egress request matching url {url_substr:?} has header {header_name:?} \
with the expected value (redacted, not logged); header names present (first \
matching request): {seen:?}"
)
.into())
}

/// Assert some recorded capability result (tool output) — i.e. a surfaced
/// HTTP response — serializes to text containing `needle`. Proves the keyed
/// scripted body actually surfaced back to the model as a tool result.
Expand Down
Loading
Loading