cherry-pick(pr-9744): test(integration): add general live-test tool for the real "default" combo + rootless wire capture

scripts/sre/tcp-close-analyzer.py

@@ -17,8 +17,8 @@
 for this one question; not a general-purpose pcap toolkit.
 
 ────────────────────────────────────────────────────────────────────────────
-CAPTURING (run this yourself — needs root/sudo for CAP_NET_RAW; also see
---show-capture-cmd)
+CAPTURING (run this yourself — needs root/sudo for CAP_NET_RAW, UNLESS you
+use the rootless method below; also see --show-capture-cmd)
 ────────────────────────────────────────────────────────────────────────────
 
 Rootless Podman gotcha: there is usually NO `podman3`/`podmanN` bridge
@@ -33,6 +33,24 @@
     sudo nsenter -t "$PID" -n tcpdump -i any -w /tmp/omniroute-capture.pcap \\
         'host <omniroute-container-ip> and port 20128'
 
+Rootless alternative (NO sudo needed): a bare `nsenter -t $PID -n` fails
+with "Invalid argument" for a rootless container, because its network
+namespace lives inside a user namespace you're not in yet. `podman unshare`
+puts you in that same user namespace first, so `nsenter --net=` against the
+container's netns path succeeds as a plain user — verified working live
+(captured a real `POST /v1/chat/completions` request body in cleartext this
+way, no root at any point):
+
+    NETNS=$(podman inspect omniroute-dev --format '{{.NetworkSettings.SandboxKey}}')
+    podman unshare nsenter --net="$NETNS" -- \\
+        tcpdump -i any -w /tmp/omniroute-capture.pcap 'port 20128'
+
+No `sudo chmod` needed afterward either, since the file was never
+root-owned. This is also what
+tests/integration/wireCapture.ts + liveContainerHarness.ts automate for the
+live wire-capture test suite (its own dedicated throwaway container, not
+omniroute-dev) — see RUN_LIVE_WIRE_CAPTURE=1 in that test file.
+
 Find the container's IP first with:
     podman inspect omniroute-dev --format '{{.NetworkSettings.Networks}}'
 

tests/integration/live-default-combo-wire-capture.test.ts

@@ -0,0 +1,143 @@
+/**
+ * tests/integration/live-default-combo-wire-capture.test.ts
+ *
+ * Wire-level correlation test. Spins up a dedicated, throwaway podman
+ * container (liveContainerHarness.ts), captures its network traffic
+ * (wireCapture.ts — rootless tcpdump via `podman unshare nsenter`, no root),
+ * sends a representative sample of requests against the real "default"
+ * combo, then cross-checks each request's app-level result (JSON status)
+ * against what actually went out on the wire (HTTP response status line,
+ * verdict on who closed the connection first). Catches bugs where the app
+ * layer claims success but the wire shows a truncated/reset stream.
+ *
+ * Fully self-contained — does not touch omniroute-beta or omniroute-dev
+ * (only reads from omniroute-dev's DB once, to seed its own dedicated
+ * container's data dir). Gated on RUN_LIVE_WIRE_CAPTURE=1: needs podman,
+ * tcpdump, python3, and a real .env with provider credentials, so it must
+ * never run in CI.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  LIVE_CONTAINER_ENABLED,
+  startLiveContainer,
+  type LiveContainerHandle,
+} from "./liveContainerHarness.ts";
+import {
+  startWireCapture,
+  analyzeCapture,
+  indexByCorrelationId,
+  responseStatusLine,
+  type CaptureHandle,
+} from "./wireCapture.ts";
+import {
+  getDefaultComboModelTargets,
+  filterActiveModelTargets,
+  sendModelRequest,
+} from "./liveDefaultComboShared.ts";
+
+const skip = !LIVE_CONTAINER_ENABLED
+  ? "RUN_LIVE_WIRE_CAPTURE not set — skipping wire-capture live test"
+  : undefined;
+
+// Wire-level correlation is the point of this suite, not breadth across
+// every provider (already covered by live-default-combo-workload.test.ts) —
+// keep the sample small so capture/analysis stays fast.
+const SAMPLE_SIZE = 4;
+
+let container: LiveContainerHandle;
+let capture: CaptureHandle;
+
+test.before(async () => {
+  if (skip) return;
+  container = await startLiveContainer();
+  process.env.DATA_DIR = container.dataDir;
+
+  // PID-scoped so a concurrent session running this same test never
+  // collides on the capture file or the pkill-by-path cleanup in
+  // wireCapture.ts's stop().
+  const pcapPath = `/tmp/omniroute-live-wire-capture-${process.pid}.pcap`;
+  // Capture happens INSIDE the container's own netns (podman unshare
+  // nsenter --net=<SandboxKey>), so packets there are addressed to the
+  // container's internal listening port (20128), not the dynamically
+  // assigned host port used to reach it from outside — filtering on
+  // hostPort here would silently match nothing.
+  capture = await startWireCapture(container.netnsPath, pcapPath, "tcp port 20128");
+});
+
+test.after(async () => {
+  if (skip) return;
+  await capture?.stop();
+  await container?.stop();
+});
+
+test(
+  "wire capture: app-level status matches the HTTP status line actually observed on the wire",
+  { skip },
+  async () => {
+    const allTargets = await getDefaultComboModelTargets();
+    assert.ok(allTargets.length > 0, `"default" combo has no model steps — nothing to test`);
+
+    const { active } = await filterActiveModelTargets(allTargets, {
+      baseUrl: container.baseUrl,
+      apiKey: container.managementApiKey,
+    });
+    assert.ok(active.length > 0, "no active provider connections in the seeded container");
+
+    const sample = active.slice(0, SAMPLE_SIZE);
+    console.log(
+      `\n  [wire-capture] sampling ${sample.length} model(s): ${sample.map((t) => t.model).join(", ")}`
+    );
+
+    const results = await Promise.all(
+      sample.map((t) =>
+        sendModelRequest(t.model, false, "chat", {
+          baseUrl: container.baseUrl,
+          apiKey: container.apiKey,
+        })
+      )
+    );
+
+    // Give the capture a moment to flush the last packets before analyzing.
+    await new Promise((r) => setTimeout(r, 1000));
+    await capture.stop();
+    const streams = await analyzeCapture(capture.pcapPath);
+    const byCorrelationId = indexByCorrelationId(streams);
+
+    console.log(`  [wire-capture] captured ${streams.length} TCP stream(s)`);
+
+    const mismatches: string[] = [];
+    for (const r of results) {
+      if (r.correlationId === "?") {
+        mismatches.push(`${r.model}: no correlationId returned in response headers`);
+        continue;
+      }
+      const matched = byCorrelationId.get(r.correlationId);
+      if (!matched || matched.length === 0) {
+        mismatches.push(
+          `${r.model}: correlationId ${r.correlationId} not found in any captured wire stream`
+        );
+        continue;
+      }
+      const wireStatusLines = matched.map(responseStatusLine).filter(Boolean);
+      const wireStatusCodes = wireStatusLines.map((line) => line!.split(" ")[1]);
+      if (!wireStatusCodes.includes(String(r.status))) {
+        mismatches.push(
+          `${r.model}: app-level status ${r.status} but wire shows ${wireStatusCodes.join(",") || "no status line"} (cid ${r.correlationId})`
+        );
+      }
+    }
+
+    if (mismatches.length > 0) {
+      console.log(`\n  Wire/app-level mismatches (${mismatches.length}):`);
+      for (const m of mismatches) console.log(`    ${m}`);
+    }
+
+    assert.equal(
+      mismatches.length,
+      0,
+      `${mismatches.length}/${results.length} requests had app-level results that don't match what was observed on the wire`
+    );
+  }
+);

tests/integration/live-default-combo-workload.test.ts

@@ -0,0 +1,113 @@
+/**
+ * tests/integration/live-default-combo-workload.test.ts
+ *
+ * General breadth test against the REAL, currently-configured "default"
+ * combo on the target instance — unlike live-gemini-workload.test.ts (which
+ * provisions its own narrow 2-model Gemini-only combo), this targets every
+ * provider/model step the operator actually has in "default" directly,
+ * bypassing combo routing. One request per configured model: non-streaming
+ * + streaming Chat Completions, and streaming Responses API. Skips (never
+ * fails) any model whose provider connection isn't currently active, so one
+ * unrelated provider outage doesn't block the rest of the run.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  skip,
+  getDefaultComboModelTargets,
+  filterActiveModelTargets,
+  sendModelRequest,
+} from "./liveDefaultComboShared.ts";
+
+let modelNames: string[] = [];
+
+test.before(async () => {
+  if (skip) return;
+  const targets = await getDefaultComboModelTargets();
+  assert.ok(targets.length > 0, `"default" combo has no model steps — nothing to test`);
+
+  const { active, skipped } = await filterActiveModelTargets(targets);
+  if (skipped.length > 0) {
+    console.log(`\n  [setup] skipping ${skipped.length} model(s) with inactive provider:`);
+    for (const s of skipped) console.log(`    - ${s}`);
+  }
+
+  modelNames = active.map((t) => t.model);
+  console.log(`\n  [setup] testing ${modelNames.length} model(s) from the live "default" combo`);
+});
+
+test(
+  "[32] default combo: non-streaming chat completions across every configured model",
+  { skip },
+  async () => {
+    const failures: string[] = [];
+    for (const model of modelNames) {
+      const r = await sendModelRequest(model, false, "chat");
+      if (r.status !== 200 || r.contentLength === 0) {
+        failures.push(
+          `${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars`
+        );
+      }
+    }
+    if (failures.length > 0) {
+      console.log(`\n  Non-streaming failures (${failures.length}/${modelNames.length}):`);
+      for (const f of failures) console.log(`    ${f}`);
+    }
+    assert.equal(
+      failures.length,
+      0,
+      `${failures.length}/${modelNames.length} models failed non-streaming chat`
+    );
+  }
+);
+
+test(
+  "[33] default combo: streaming chat completions across every configured model",
+  { skip },
+  async () => {
+    const failures: string[] = [];
+    for (const model of modelNames) {
+      const r = await sendModelRequest(model, true, "chat");
+      if (r.status !== 200 || r.contentLength === 0) {
+        failures.push(
+          `${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars`
+        );
+      }
+    }
+    if (failures.length > 0) {
+      console.log(`\n  Streaming failures (${failures.length}/${modelNames.length}):`);
+      for (const f of failures) console.log(`    ${f}`);
+    }
+    assert.equal(
+      failures.length,
+      0,
+      `${failures.length}/${modelNames.length} models failed streaming chat`
+    );
+  }
+);
+
+test(
+  "[34] default combo: streaming responses API across every configured model",
+  { skip },
+  async () => {
+    const failures: string[] = [];
+    for (const model of modelNames) {
+      const r = await sendModelRequest(model, true, "responses");
+      if (r.status !== 200 || r.contentLength === 0) {
+        failures.push(
+          `${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars`
+        );
+      }
+    }
+    if (failures.length > 0) {
+      console.log(`\n  Responses API failures (${failures.length}/${modelNames.length}):`);
+      for (const f of failures) console.log(`    ${f}`);
+    }
+    assert.equal(
+      failures.length,
+      0,
+      `${failures.length}/${modelNames.length} models failed streaming Responses API`
+    );
+  }
+);