diff --git a/.github/workflows/iroh-release-gate.yml b/.github/workflows/iroh-release-gate.yml index 9150f6c42cba..54395665af36 100644 --- a/.github/workflows/iroh-release-gate.yml +++ b/.github/workflows/iroh-release-gate.yml @@ -365,10 +365,10 @@ jobs: chmod 600 "$production_env" - name: Run Iroh gate - # Every non-soak phase is capped at 25 minutes. Stress soaks retain - # their one-hour workload window while the gate script bounds each - # setup, build, and launch phase separately. - timeout-minutes: ${{ inputs.soak_profile == 'stress' && 75 || 25 }} + # Relay-only stress is one hour of use plus the 30-minute credential + # rollover observation and bounded cleanup. The script's report + # timeout is 5,850 seconds, so this step must be longer than that. + timeout-minutes: ${{ inputs.soak_profile == 'stress' && 125 || 25 }} env: # The hosted runner does not have cmuxterm-hq's tagged-backend # helper. The explicit Worker origin remains the app's API and v2 @@ -379,6 +379,9 @@ jobs: # has no cmux-tui artifact yet, use the newest older artifact with # identical client inputs instead of failing before the app starts. CMUX_TUI_CLIENT_MAX_FALLBACK: 5 + CMUX_CODEX_MODEL: gpt-5.3-codex-spark + CMUX_CODEX_DURATION_SECONDS: "3600" + CMUX_CODEX_STRICT_MODEL: "1" run: | set -euo pipefail case "${{ matrix.mode }}" in @@ -408,11 +411,45 @@ jobs: if [[ "${{ matrix.mode }}" == automatic || "${{ matrix.mode }}" == relay-only ]]; then GATE_ARGS+=(--soak-profile "${{ inputs.soak_profile }}") fi - if [[ "${{ inputs.environment }}" == production && "${{ matrix.mode }}" == relay-only && "${{ inputs.soak_profile }}" == stress ]]; then + if [[ "${{ inputs.soak_profile }}" == stress && ( "${{ matrix.mode }}" == automatic || "${{ matrix.mode }}" == relay-only ) ]]; then GATE_ARGS+=(--real-usage) fi ./scripts/run-iroh-release-gate.sh "${GATE_ARGS[@]}" + - name: Verify canonical v2 Worker identity + if: ${{ always() }} + env: + EXPECTED_ENVIRONMENT: ${{ inputs.environment }} + EXPECTED_V2_URL: ${{ inputs.environment == 'production' && 'https://cmux-v2.debussy.workers.dev' || inputs.v2_base_url }} + HEALTH_OUTPUT: ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-worker-health.json + run: | + set -euo pipefail + curl --fail --silent --show-error --retry 3 --retry-connrefused \ + --connect-timeout 10 --max-time 30 \ + "$EXPECTED_V2_URL/v2/health" > "$HEALTH_OUTPUT" + EXPECTED_ENVIRONMENT="$EXPECTED_ENVIRONMENT" EXPECTED_V2_URL="$EXPECTED_V2_URL" HEALTH_OUTPUT="$HEALTH_OUTPUT" \ + /usr/bin/python3 - <<'PY' + import json + import os + from urllib.parse import urlparse + + with open(os.environ["HEALTH_OUTPUT"], encoding="utf-8") as handle: + health = json.load(handle) + expected_environment = os.environ["EXPECTED_ENVIRONMENT"] + expected_url = os.environ["EXPECTED_V2_URL"] + if health.get("schemaId") != "health.v1": + raise SystemExit("Worker health schema is not health.v1") + if health.get("environment") != expected_environment: + raise SystemExit("Worker health environment does not match the gate") + if not isinstance(health.get("sourceRevision"), str) or not health["sourceRevision"] or health["sourceRevision"] == "unknown": + raise SystemExit("Worker health did not identify its source revision") + if health.get("storage", {}).get("maxSchemaVersion", 0) < 7: + raise SystemExit("Worker storage schema is older than v2") + if urlparse(expected_url).hostname in {"cmux-iroh-v2.debussy.workers.dev", "cmux-iroh-v2-staging.debussy.workers.dev"}: + raise SystemExit("release gate used a compatibility alias instead of the canonical Worker") + print(json.dumps({"url": expected_url, "health": health}, sort_keys=True)) + PY + - name: Upload redacted verdict if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -426,6 +463,9 @@ jobs: ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-ios-*.jsonl ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-ios-*.log ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-mac-failure.cmuxdiag + ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-latency.state* + ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-latency.json + ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-worker-health.json ${{ runner.temp }}/iroh-release-gate-${{ matrix.mode }}-real-usage/** if-no-files-found: warn retention-days: 7 diff --git a/CLI/CMUXCLI+AgentHookAdmission.swift b/CLI/CMUXCLI+AgentHookAdmission.swift index 2d15b86255a0..2c5f118743e8 100644 --- a/CLI/CMUXCLI+AgentHookAdmission.swift +++ b/CLI/CMUXCLI+AgentHookAdmission.swift @@ -565,7 +565,7 @@ extension CMUXCLI { ) } changed = true - case .unavailable, .pending, .healthy: + case .unavailable, .pending, .aborted, .healthy: break } } diff --git a/CLI/CodexTranscriptFailureReadResult.swift b/CLI/CodexTranscriptFailureReadResult.swift index 08abfc6a28bf..c3330d6c965a 100644 --- a/CLI/CodexTranscriptFailureReadResult.swift +++ b/CLI/CodexTranscriptFailureReadResult.swift @@ -1,6 +1,7 @@ enum CodexTranscriptFailureReadResult { case unavailable case pending + case aborted case healthy(lastAssistantMessage: String?) case failure(CodexHookFailureCandidate) } diff --git a/CLI/CodexTranscriptMonitorStopReplay.swift b/CLI/CodexTranscriptMonitorStopReplay.swift index 86941f904b75..72bc79dc5836 100644 --- a/CLI/CodexTranscriptMonitorStopReplay.swift +++ b/CLI/CodexTranscriptMonitorStopReplay.swift @@ -6,6 +6,7 @@ struct CodexTranscriptMonitorStopReplay { let payload: String let workspaceId: String let surfaceId: String? + let suppressNotification: Bool init?( sessionId: String, @@ -13,11 +14,13 @@ struct CodexTranscriptMonitorStopReplay { transcriptPath: String?, workspaceId: String, surfaceId: String?, - lastAssistantMessage: String? + lastAssistantMessage: String?, + suppressNotification: Bool = false ) { guard !sessionId.isEmpty, !workspaceId.isEmpty else { return nil } self.workspaceId = workspaceId self.surfaceId = surfaceId + self.suppressNotification = suppressNotification var object: [String: Any] = [ "session_id": sessionId, diff --git a/CLI/cmux.swift b/CLI/cmux.swift index b2b0b1ee0a70..8a48372f2c3b 100644 --- a/CLI/cmux.swift +++ b/CLI/cmux.swift @@ -30805,6 +30805,8 @@ struct CMUXCLI { return summarizeCodexHookFailureCandidate(failure) case .healthy: return nil + case .aborted: + return nil case .pending, .unavailable: break } @@ -30818,6 +30820,8 @@ struct CMUXCLI { return summarizeCodexHookFailureCandidate(failure) case .healthy: return nil + case .aborted: + return nil case .pending, .unavailable: break } @@ -30852,6 +30856,7 @@ struct CMUXCLI { var candidateCanPublishBeforeTerminal = false var sawAssistantMessage = false var sawTerminalTurn = false + var sawAbortedTurn = false var sawRelevantTurn = turnId == nil var lastAssistantMessage: String? for line in lines { @@ -30885,6 +30890,7 @@ struct CMUXCLI { sawRelevantTurn = true candidate = nil candidateCanPublishBeforeTerminal = false + sawAbortedTurn = false case "error": let payloadTurnId = firstString(in: payload, keys: ["turn_id", "turnId"]) if let turnId, let payloadTurnId { @@ -30930,6 +30936,7 @@ struct CMUXCLI { // has no final response to classify as a failure, so let the // normal Stop replay retire its stale prompt record. if eventType == "turn_aborted" { + sawAbortedTurn = true continue } // Codex persists fatal turn failures inside task_complete.error. Standalone @@ -30979,6 +30986,9 @@ struct CMUXCLI { if candidate != nil, turnId != nil, !sawRelevantTurn { return .pending } + if sawAbortedTurn, candidate == nil { + return .aborted + } if requireTerminalCompletion, !sawTerminalTurn { return .pending } @@ -31848,6 +31858,16 @@ struct CMUXCLI { surfaceId: surfaceId, lastAssistantMessage: lastAssistantMessage ) + case .aborted: + return CodexTranscriptMonitorStopReplay( + sessionId: sessionId, + turnId: turnId, + transcriptPath: currentTranscriptPath, + workspaceId: workspaceId, + surfaceId: surfaceId, + lastAssistantMessage: nil, + suppressNotification: true + ) case .pending: break case .unavailable: @@ -37641,6 +37661,7 @@ export default { turnID: effectiveCodexStopTurnID, workspaceID: workspaceId, surfaceID: surfaceId, + claimNotification: monitorReplay?.suppressNotification != true, // Tokenized Codex launches must not let a delayed Stop // for an older turn settle the currently active turn. // Legacy unwrapped launches retain their historical @@ -37700,6 +37721,7 @@ export default { } } let suppressCompletionNotification = suppressVisibleMutations + || monitorReplay?.suppressNotification == true || codexHasActiveBackgroundWork let cursorStopApprovalNotificationKeys: [String] = { guard def.name == "cursor", !sessionId.isEmpty else { return [] } diff --git a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/JumpToBottomAffordanceTests.swift b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/JumpToBottomAffordanceTests.swift index 12c9c8c7d4de..912e4b38b99a 100644 --- a/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/JumpToBottomAffordanceTests.swift +++ b/Packages/macOS/CmuxFoundation/Tests/CmuxFoundationTests/JumpToBottomAffordanceTests.swift @@ -46,8 +46,10 @@ import Testing ownerChecks += 1 return true } - #expect(!affordance.update(snapshot(below: 8), isEnabled: true, contentOwnsScrolling: ownsScrolling)) - #expect(!affordance.update(snapshot(below: 9), isEnabled: true, contentOwnsScrolling: ownsScrolling)) + let firstChanged = affordance.update(snapshot(below: 8), isEnabled: true, contentOwnsScrolling: ownsScrolling) + let secondChanged = affordance.update(snapshot(below: 9), isEnabled: true, contentOwnsScrolling: ownsScrolling) + #expect(!firstChanged) + #expect(!secondChanged) #expect(!affordance.isVisible) // The owner is asked once per departure from the bottom, not per snapshot. #expect(ownerChecks == 1) diff --git a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift index 0d2e96d5824f..c8282563cb28 100644 --- a/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift +++ b/Packages/macOS/CmuxRemoteSession/Tests/CmuxRemoteSessionTests/RemoteRelaySlotTeardownTests.swift @@ -293,7 +293,7 @@ struct RemoteRelaySlotTeardownTests { let succeeded = await coordinator.stopAndWait(cleanupScope: .persistentSlot) - let cleanupCommand = try #require(runner.requests.last?.arguments.last) + let cleanupCommand = try command(in: runner) { $0.contains("serve --persistent-stop --slot") } #expect(succeeded) #expect(cleanupCommand.contains("serve --persistent-stop --slot")) #expect(cleanupCommand.contains("64010.shell")) @@ -316,7 +316,7 @@ struct RemoteRelaySlotTeardownTests { let succeeded = await coordinator.stopAndWait(cleanupScope: .transport) - let cleanupCommand = try #require(runner.requests.last?.arguments.last) + let cleanupCommand = try command(in: runner) { $0.contains("64010.slot") } #expect(succeeded) #expect(!cleanupCommand.contains("serve --persistent-stop --slot")) #expect(!cleanupCommand.contains("rm -rf")) @@ -335,7 +335,7 @@ struct RemoteRelaySlotTeardownTests { let succeeded = await coordinator.stopAndWait(cleanupScope: .persistentSlot) - let cleanupCommand = try #require(runner.requests.last?.arguments.last) + let cleanupCommand = try command(in: runner) { $0.contains("serve --persistent-stop --slot") } #expect(succeeded) #expect(cleanupCommand.contains("$HOME/.cmux/bin/cmuxd-remote")) #expect(cleanupCommand.contains("serve --persistent-stop --slot")) @@ -355,9 +355,9 @@ struct RemoteRelaySlotTeardownTests { #expect(succeeded) #expect(runner.requests.count == 2) - let metadataCleanup = try #require(runner.requests.first?.arguments.last) + let metadataCleanup = try command(in: runner) { $0.contains("64010.slot") } #expect(metadataCleanup.contains("64010.slot")) - let directCleanup = try #require(runner.requests.last?.arguments.last) + let directCleanup = try command(in: runner) { $0.contains("$HOME/.cmux/bin/cmuxd-remote") } #expect(directCleanup.contains("$HOME/.cmux/bin/cmuxd-remote")) #expect(directCleanup.contains("serve --persistent-stop --slot")) #expect(!directCleanup.contains("relay_socket=")) @@ -410,6 +410,19 @@ struct RemoteRelaySlotTeardownTests { ) } + private func command( + in runner: SpyProcessRunner, + matching predicate: (String) -> Bool + ) throws -> String { + let commands = runner.requests.compactMap(\.arguments.last) + guard let command = commands.first(where: { predicate($0) }) else { + throw MissingCleanupCommand() + } + return command + } + + private struct MissingCleanupCommand: Error {} + enum MalformedSlotCleanupScope: String, CaseIterable, Sendable { case persistentSlot case transport diff --git a/Packages/macOS/CmuxSettings/Sources/CmuxSettings/Resources/CustomSidebarTemplates/agents-board.js b/Packages/macOS/CmuxSettings/Sources/CmuxSettings/Resources/CustomSidebarTemplates/agents-board.js index 5621a4f0e8e5..524fcf1b1fb0 100644 --- a/Packages/macOS/CmuxSettings/Sources/CmuxSettings/Resources/CustomSidebarTemplates/agents-board.js +++ b/Packages/macOS/CmuxSettings/Sources/CmuxSettings/Resources/CustomSidebarTemplates/agents-board.js @@ -1,7 +1,7 @@ // agents-board: subagents grouped by STATUS, attention first. The question // this layout answers is "what needs me right now" - needs-input sessions // get the loudest section at the top, everything else stays quiet. -// cmux sidebar set custom agents-board +// cmux left-sidebar set custom agents-board const STATUS_META = { needs_input: { label: "NEEDS YOU", color: "#FF9F0A", strong: true }, diff --git a/Packages/macOS/CmuxUpdaterUI/Package.swift b/Packages/macOS/CmuxUpdaterUI/Package.swift index e6874c091e38..99e5e1e1d0d5 100644 --- a/Packages/macOS/CmuxUpdaterUI/Package.swift +++ b/Packages/macOS/CmuxUpdaterUI/Package.swift @@ -14,6 +14,7 @@ let package = Package( ), ], dependencies: [ + .package(path: "../CmuxAppKitSupportUI"), .package(path: "../CmuxFoundation"), .package(path: "../CmuxUpdater"), .package(url: "https://github.com/sparkle-project/Sparkle", from: "2.9.0"), @@ -23,6 +24,7 @@ let package = Package( name: "CmuxUpdaterUI", dependencies: [ "CmuxFoundation", + .product(name: "CmuxAppKitSupportUI", package: "CmuxAppKitSupportUI"), "CmuxUpdater", .product(name: "Sparkle", package: "Sparkle"), ], @@ -34,7 +36,10 @@ let package = Package( ), .testTarget( name: "CmuxUpdaterUITests", - dependencies: ["CmuxUpdaterUI"], + dependencies: [ + "CmuxUpdaterUI", + .product(name: "CmuxAppKitSupportUI", package: "CmuxAppKitSupportUI"), + ], swiftSettings: [ .swiftLanguageMode(.v6), .enableUpcomingFeature("ExistentialAny"), diff --git a/Packages/macOS/CmuxUpdaterUI/Sources/CmuxUpdaterUI/UpdateBadge.swift b/Packages/macOS/CmuxUpdaterUI/Sources/CmuxUpdaterUI/UpdateBadge.swift index 2c02a00ef022..5afe16794081 100644 --- a/Packages/macOS/CmuxUpdaterUI/Sources/CmuxUpdaterUI/UpdateBadge.swift +++ b/Packages/macOS/CmuxUpdaterUI/Sources/CmuxUpdaterUI/UpdateBadge.swift @@ -1,5 +1,7 @@ public import SwiftUI +import CmuxAppKitSupportUI public import CmuxUpdater +import AppKit /// A badge view that displays the current state of an update operation (icon, progress ring, /// or loading spinner) for the update pill. @@ -7,6 +9,10 @@ public struct UpdateBadge: View { private let model: UpdateStateModel private let appearance: UpdateAppearance + private static let iconSize: CGFloat = 14 + private static let iconPointSize: CGFloat = 13 + private static let iconWeight: NSFont.Weight = .semibold + /// Creates a badge for `model`, using `appearance` for the loading-spinner tint. public init(model: UpdateStateModel, appearance: UpdateAppearance) { self.model = model @@ -22,7 +28,7 @@ public struct UpdateBadge: View { private var badgeContent: some View { if model.showsDetectedBackgroundUpdate { if let iconName = model.iconName { - Image(systemName: iconName) + badgeImage(iconName) } } else { switch model.effectiveState { @@ -31,7 +37,7 @@ public struct UpdateBadge: View { let progress = min(1, max(0, Double(download.progress) / Double(expectedLength))) ProgressRingView(progress: progress) } else { - Image(systemName: "arrow.down.circle") + badgeImage("arrow.down.circle") } case .extracting(let extracting): @@ -42,11 +48,39 @@ public struct UpdateBadge: View { default: if let iconName = model.iconName { - Image(systemName: iconName) + badgeImage(iconName) } } } } + + /// Builds the AppKit-owned request used by every updater symbol. + @MainActor + static func hostedIconRequest( + systemName: String, + tintColor: NSColor + ) -> CmuxResolvedIconRequest { + CmuxResolvedIconRequest( + source: .systemSymbol(name: systemName, accessibilityDescription: nil), + size: NSSize(width: iconSize, height: iconSize), + tintColor: tintColor, + symbolWeight: iconWeight, + fallbackSource: .systemSymbol( + name: systemName, + accessibilityDescription: nil + ), + symbolPointSize: iconPointSize + ) + } + + private func badgeImage(_ systemName: String) -> some View { + CmuxResolvedIconImage(request: Self.hostedIconRequest( + systemName: systemName, + tintColor: NSColor(appearance.foregroundColor(for: model)) + )) + .frame(width: Self.iconSize, height: Self.iconSize) + .accessibilityHidden(true) + } } private struct ProgressRingView: View { diff --git a/docs/iroh-v2/IMPLEMENTATION.md b/docs/iroh-v2/IMPLEMENTATION.md index 6764f34124b3..5b0c622ea301 100644 --- a/docs/iroh-v2/IMPLEMENTATION.md +++ b/docs/iroh-v2/IMPLEMENTATION.md @@ -8,7 +8,7 @@ Implement the full backend, database, Mac/iOS clients, shared contracts, Dashboa **Lawrence's adopted decisions are implementation requirements.** Use Zod instead of Ajv for server input and output validation; export JSON Schema and generate Swift/TypeScript models with quicktype. HTTP and socket requests share one local operation handler. Apply immutable Drizzle migrations before serving requests, enforce storage bounds atomically in SQLite, handle storage failures explicitly, and run real workerd migration/persistence tests. Required release checks enforce the Cloudflare boundary and contract compatibility. Deploy in stages with complete, bounded observability. The [adopted rules](design/IROH-DECISIONS.md#accepted-backend-implementation-rules) and [source assessment](design/PR-12199-LESSONS.md) retain the full details and our adaptations, including no scheduled challenge cleanup. -Acceptance requires sustained real simulator use, including relay-only/high-latency traffic, credential rollover without session interruption, multiple workspaces and active Codex work using inexpensive Codex models, launch-to-workspace-list under 3.5 seconds, and resume within 2 seconds after 2 minutes backgrounded. Record one-hour runs against the deployed implementation, with environment, build, connection and timing evidence. The acceptance document supplies the complete matrix. +Acceptance requires sustained real simulator use, including relay-only/high-latency traffic, credential rollover without session interruption, multiple workspaces and active Codex work using `gpt-5.3-codex-spark`, launch-to-workspace-list under 2.5 seconds, and resume within 2 seconds after 2 minutes backgrounded. Record one-hour runs against the deployed implementation, with environment, build, connection and timing evidence. The acceptance document supplies the complete matrix. ## Work and evidence @@ -77,7 +77,7 @@ Acceptance requires sustained real simulator use, including relay-only/high-late - A previous simulator paired and exchanged data but used an older dirty artifact. Its brief recording and passive uptime do not satisfy the required engaged hour. The clean simulator's setup was blocked by the database error above. The sustained acceptance run has not passed. - The clean simulator established same-account RPC after the database fix. The readiness helper incorrectly records the current checkout SHA even when the installed app is older; binary hashes and baked build metadata remain authoritative. The old first-frame timeout on admitted IRX peers is fixed in `5dc1bb65020`; unauthenticated transports retain the fifteen-second limit. A behavior regression waits for that legacy timeout, then verifies the admitted peer can send its first request. New builds and live verification are required. - A debug-only environment switch, `CMUX_IROH_V2_VERIFY_RENEW_INTERVAL_SECONDS` (30 to 900 seconds), schedules the same API-ticket, relay-credential and directory renewal methods used in production. It exists only in Debug builds for the shortened 15-minute verification and leaves production lifetimes unchanged. -- The configured account rejects `gpt-5.3-codex-spark`. The user requested inexpensive Codex models; the acceptance author had unnecessarily narrowed that to one exact model. The criteria now follow the user request and require the actual selected model and active session evidence. +- The stress gate requires three real `gpt-5.3-codex-spark` sessions and fails closed when the runner cannot authenticate or select that model. The runner must therefore provide an authorized Codex login with access to that model; a fallback model is not accepted as proof. ## Historical Mac build checkpoint diff --git a/scripts/ci/restore-app-host-test-product.sh b/scripts/ci/restore-app-host-test-product.sh index e13e742900b9..22fc045303d7 100755 --- a/scripts/ci/restore-app-host-test-product.sh +++ b/scripts/ci/restore-app-host-test-product.sh @@ -110,3 +110,9 @@ if [ -n "${GITHUB_ENV:-}" ]; then echo "CMUX_CI_RUNTIME_SOURCE_ROOT=$CMUX_CI_RUNTIME_SOURCE_ROOT" >> "$GITHUB_ENV" fi scripts/ci/canonical-build-root.sh --runtime-source "$PWD" +# `#filePath` literals keep the producer's canonical path even though the +# compiler's prefix map makes the rest of the test metadata portable. Keep a +# second alias at that exact path for tests that still open repository files +# directly (for example bundled CLI scripts). +CMUX_CI_RUNTIME_SOURCE_ROOT="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" \ + scripts/ci/canonical-build-root.sh --runtime-source "$PWD" diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 05d1cd108499..f59f5ab75e98 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -48,7 +48,7 @@ relay-only defaults for both installed app bundles before `mobile-dev-launch.sh` starts the simulator. The driver assumes that policy is already configured; it does not switch transport modes during a step. -`iroh-codex-workload.sh` starts three real `codex --yolo -m gpt-5.5-mini` sessions in separate Mac workspaces and two supporting workspaces. It records workspace, surface, model, and observed output markers in `codex-workload.jsonl`; set `CMUX_CODEX_DURATION_SECONDS` to keep the sessions active while the iOS gate runs. +`iroh-codex-workload.sh` starts three real `codex --yolo -m gpt-5.3-codex-spark` sessions in separate Mac workspaces and two supporting workspaces. It records workspace, surface, model, and observed output markers in `codex-workload.jsonl`; set `CMUX_CODEX_DURATION_SECONDS` to keep the sessions active while the iOS gate runs. Strict release verification fails if the requested model is unavailable instead of silently substituting another model. On failure exit nonzero and print `E2E FAIL step=` as the last stderr line, where `` is a step id below or `sign-in`, `pair`, `connect` for setup. diff --git a/scripts/e2e/iroh-codex-workload.sh b/scripts/e2e/iroh-codex-workload.sh index ae7cb1c46300..06ac1338e010 100755 --- a/scripts/e2e/iroh-codex-workload.sh +++ b/scripts/e2e/iroh-codex-workload.sh @@ -5,10 +5,11 @@ set -euo pipefail TAG="${CMUX_E2E_TAG:-}" EVIDENCE_DIR="${CMUX_CODEX_EVIDENCE_DIR:-}" -MODEL="${CMUX_CODEX_MODEL:-gpt-5.5-mini}" -DURATION_SECONDS="${CMUX_CODEX_DURATION_SECONDS:-900}" +MODEL="${CMUX_CODEX_MODEL:-gpt-5.3-codex-spark}" +DURATION_SECONDS="${CMUX_CODEX_DURATION_SECONDS:-3600}" COUNT="${CMUX_CODEX_SESSION_COUNT:-3}" SHUTDOWN_FILE="${CMUX_CODEX_SHUTDOWN_FILE:-}" +STRICT_MODEL="${CMUX_CODEX_STRICT_MODEL:-1}" [[ -n "$TAG" && -n "$EVIDENCE_DIR" ]] || { echo "Usage: CMUX_E2E_TAG= CMUX_CODEX_EVIDENCE_DIR= $0" >&2 exit 2 @@ -17,6 +18,10 @@ SHUTDOWN_FILE="${CMUX_CODEX_SHUTDOWN_FILE:-}" echo "error: duration and session count must be positive integers" >&2 exit 2 } +if [[ "$STRICT_MODEL" == "1" && "$MODEL" != "gpt-5.3-codex-spark" ]]; then + echo "error: strict verification requires gpt-5.3-codex-spark, got '$MODEL'" >&2 + exit 2 +fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" @@ -25,10 +30,7 @@ mkdir -p "$EVIDENCE_DIR" LOG="$EVIDENCE_DIR/codex-workload.jsonl" : > "$LOG" -# The preferred model is unavailable to the ChatGPT account used by this -# hosted verification lane. Record the explicit fallback in the evidence so a -# successful run proves the workload ran with a real supported Codex model. -printf '{"event":"model_selection","requested_preferred":"gpt-5.3-codex-spark","selected":"%s","unavailable_reason":"unsupported ChatGPT account"}\n' "$MODEL" >> "$LOG" +printf '{"event":"model_selection","requested":"gpt-5.3-codex-spark","selected":"%s","strict":%s}\n' "$MODEL" "$([[ "$STRICT_MODEL" == "1" ]] && printf true || printf false)" >> "$LOG" json_value() { /usr/bin/python3 -c 'import json,sys; d=json.load(sys.stdin); v=d.get(sys.argv[1]); print(v if v is not None else "")' "$1" @@ -91,6 +93,7 @@ declare -a ITERATION_COUNTS=() declare -a LOG_OFFSETS=() declare -a LOG_TAILS=() declare -a ITERATION_MARKERS=() +declare -a LAST_ACTIVITY_EPOCHS=() # Closing a workspace terminates the terminal process group that owns the # Codex/support command. Always clean up, including when a marker or RPC check @@ -137,6 +140,7 @@ for ((index=1; index<=COUNT+2; index++)); do LOG_OFFSETS+=(0) LOG_TAILS+=("") ITERATION_MARKERS+=("") + LAST_ACTIVITY_EPOCHS+=("$(date +%s)") printf '{"event":"session_started","role":"%s","index":%d,"workspace_id":"%s","surface_id":"%s","model":"%s","working_directory":"%s","started_at":"%s"}\n' \ "$role" "$index" "$workspace" "$surface" "$MODEL" "$workdir" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$LOG" done @@ -151,11 +155,15 @@ while (( $(date +%s) < deadline )); do marker_seen=0 if (( session_number <= COUNT )); then ready_marker="CMUX_CODEX_${session_number}_READY" + previous_log_size="${LOG_OFFSETS[$index]}" log_scan="$(scan_session_log "$session_log" "${LOG_OFFSETS[$index]}" "$session_number" "${LOG_TAILS[$index]}")" IFS='|' read -r log_size log_ready log_markers log_error <<<"$log_scan" if [[ "$log_size" =~ ^[0-9]+$ ]]; then LOG_OFFSETS[$index]="$log_size" LOG_TAILS[$index]="$(tail -c 256 "$session_log" 2>/dev/null || true)" + if (( log_size > previous_log_size )); then + LAST_ACTIVITY_EPOCHS[$index]="$(date +%s)" + fi fi if [[ "$log_ready" == "1" ]] || grep -qF "$ready_marker" <<<"$combined_output"; then READY_SESSIONS[$index]=1 @@ -190,6 +198,11 @@ while (( $(date +%s) < deadline )); do echo "error: Codex session $session_number exited or needs authentication before its ready marker" >&2 exit 1 fi + now="$(date +%s)" + if (( now - LAST_ACTIVITY_EPOCHS[index] > 300 )); then + echo "error: Codex session $session_number produced no log activity for more than five minutes" >&2 + exit 1 + fi elif grep -qF "CMUX_SUPPORT_" <<<"$combined_output"; then marker_seen=1 fi diff --git a/scripts/e2e/iroh-latency-impairment.sh b/scripts/e2e/iroh-latency-impairment.sh new file mode 100755 index 000000000000..d75d16bb94a9 --- /dev/null +++ b/scripts/e2e/iroh-latency-impairment.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 start|stop " >&2 + exit 2 +} + +[[ $# -eq 2 ]] || usage +ACTION="$1" +STATE_FILE="$2" +# The stock macOS pf.conf evaluates the com.apple/* subtree. Keeping our +# anchor below that parent makes the rule active without changing pf.conf. +ANCHOR="com.apple/cmux-iroh-release-gate" + +require_tools() { + [[ "$(uname -s)" == "Darwin" ]] || { echo "error: latency impairment requires macOS" >&2; exit 1; } + command -v dnctl >/dev/null || { echo "error: dnctl is unavailable" >&2; exit 1; } + command -v pfctl >/dev/null || { echo "error: pfctl is unavailable" >&2; exit 1; } + sudo -n true >/dev/null 2>&1 || { + echo "error: passwordless sudo is required for the reversible latency profile" >&2 + exit 1 + } +} + +case "$ACTION" in + start) + if [[ -e "$STATE_FILE" ]]; then + if [[ -f "$STATE_FILE" ]] && grep -q '^stopped_at=' "$STATE_FILE"; then + archive="${STATE_FILE}.previous.$(date +%s).$$" + mv "$STATE_FILE" "$archive" + echo "archived completed latency state: $archive" >&2 + else + echo "error: latency state already exists and is not completed: $STATE_FILE" >&2 + exit 1 + fi + fi + require_tools + delay_ms="${CMUX_IROH_LATENCY_DELAY_MS:-150}" + [[ "$delay_ms" =~ ^[1-9][0-9]*$ ]] || { echo "error: CMUX_IROH_LATENCY_DELAY_MS must be positive" >&2; exit 2; } + pipe_id=$((300 + ($$ % 600))) + pf_was_enabled=0 + if sudo -n pfctl -s info 2>/dev/null | grep -q 'Status: Enabled'; then + pf_was_enabled=1 + fi + mkdir -p "$(dirname "$STATE_FILE")" + resources_started=0 + start_succeeded=0 + rollback_start() { + [[ "$start_succeeded" -eq 0 && "$resources_started" -eq 1 ]] || return 0 + set +e + sudo -n pfctl -a "$ANCHOR" -F all >/dev/null 2>&1 + sudo -n dnctl pipe "$pipe_id" delete >/dev/null 2>&1 + if [[ "$pf_was_enabled" -eq 0 ]]; then + sudo -n pfctl -d >/dev/null 2>&1 + fi + } + trap rollback_start EXIT + if ! sudo -n dnctl pipe "$pipe_id" config delay "${delay_ms}ms"; then + echo "error: could not configure dummynet pipe" >&2 + exit 1 + fi + resources_started=1 + if ! printf 'dummynet out proto udp from any to any pipe %s\n' "$pipe_id" | sudo -n pfctl -a "$ANCHOR" -f - >/dev/null; then + echo "error: could not install pf latency rule" >&2 + exit 1 + fi + if [[ "$pf_was_enabled" -eq 0 ]]; then + if ! sudo -n pfctl -E >/dev/null; then + echo "error: could not enable pf for latency rule" >&2 + exit 1 + fi + fi + if ! sudo -n pfctl -a "$ANCHOR" -sr 2>/dev/null | grep -Fq "dummynet out proto udp from any to any pipe $pipe_id"; then + echo "error: active pf ruleset does not contain the latency rule" >&2 + exit 1 + fi + state_tmp="${STATE_FILE}.tmp.$$" + { + printf 'pipe_id=%s\n' "$pipe_id" + printf 'delay_ms=%s\n' "$delay_ms" + printf 'pf_was_enabled=%s\n' "$pf_was_enabled" + printf 'anchor=%s\n' "$ANCHOR" + printf 'started_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "$state_tmp" + chmod 600 "$state_tmp" + if ! mv "$state_tmp" "$STATE_FILE"; then + rm -f "$state_tmp" + echo "error: could not commit latency state" >&2 + exit 1 + fi + start_succeeded=1 + trap - EXIT + echo "latency impairment active: ${delay_ms}ms UDP delay, state=$STATE_FILE" + ;; + stop) + [[ -f "$STATE_FILE" ]] || exit 0 + require_tools + # State is written by this script and contains only numeric values and the + # fixed anchor name. Read it without evaluating shell code. + pipe_id="$(awk -F= '$1 == "pipe_id" { print $2 }' "$STATE_FILE")" + pf_was_enabled="$(awk -F= '$1 == "pf_was_enabled" { print $2 }' "$STATE_FILE")" + [[ "$pipe_id" =~ ^[0-9]+$ && "$pf_was_enabled" =~ ^[01]$ ]] || { + echo "error: invalid latency state: $STATE_FILE" >&2 + exit 1 + } + stop_status=0 + if ! sudo -n pfctl -a "$ANCHOR" -F all >/dev/null 2>&1; then + stop_status=1 + fi + if ! sudo -n dnctl pipe "$pipe_id" delete >/dev/null 2>&1; then + # Deleting an already-removed pipe is safe, but an unrelated sudo or + # dummynet failure must fail the gate and remain visible. + if sudo -n dnctl pipe show 2>/dev/null | grep -Eq "(^|[[:space:]])${pipe_id}([[:space:]]|:)"; then + stop_status=1 + fi + fi + if [[ "$pf_was_enabled" -eq 0 ]]; then + if ! sudo -n pfctl -d >/dev/null 2>&1; then + stop_status=1 + fi + fi + printf 'stopped_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$STATE_FILE" + if [[ "$stop_status" -ne 0 ]]; then + echo "error: latency impairment cleanup failed; state retained at $STATE_FILE" >&2 + exit 1 + fi + echo "latency impairment removed: state=$STATE_FILE" + ;; + *) usage ;; +esac diff --git a/scripts/e2e/summarize-iroh-latency.py b/scripts/e2e/summarize-iroh-latency.py new file mode 100755 index 000000000000..1fe348326544 --- /dev/null +++ b/scripts/e2e/summarize-iroh-latency.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +if len(sys.argv) not in (4, 5): + raise SystemExit("usage: summarize-iroh-latency.py STATE OUTPUT JOURNAL_DIR [JOURNAL_PREFIX]") +state_path, output_path, journal_dir = map(Path, sys.argv[1:4]) +journal_prefix = Path(sys.argv[4]) if len(sys.argv) == 5 else None +state = {} +for line in state_path.read_text(encoding="utf-8").splitlines(): + key, separator, value = line.partition("=") + if separator: + state[key] = value +try: + applied_delay = float(state["delay_ms"]) +except (KeyError, ValueError): + raise SystemExit("latency state has no valid delay_ms") + +samples = [] +if journal_prefix is None: + journal_paths = sorted(journal_dir.glob("*-ios-iroh-v2-journal-success-*.jsonl")) +else: + journal_paths = sorted(journal_prefix.parent.glob( + journal_prefix.name + "-ios-iroh-v2-journal-success-*.jsonl" + )) +for path in journal_paths: + for raw in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + value = json.loads(raw) + except json.JSONDecodeError: + continue + stack = [value] + while stack: + item = stack.pop() + if isinstance(item, dict): + for key, child in item.items(): + # IrxJournal prefixes attributes with a_ in its JSONL + # representation. Keep one source key so a future record + # containing both spellings cannot double-count a ping. + if key == "a_rtt_ms": + try: + number = float(child) + except (TypeError, ValueError): + continue + if number >= 0: + samples.append(number) + elif isinstance(child, (dict, list)): + stack.append(child) + elif isinstance(item, list): + stack.extend(item) + +if not samples: + raise SystemExit("latency impairment was applied but no IROH RTT samples were recorded") +samples.sort() + +def percentile(percent): + index = min(len(samples) - 1, max(0, int(round((len(samples) - 1) * percent)))) + return samples[index] + +summary = { + "appliedDelayMs": applied_delay, + "sampleCount": len(samples), + "rttMs": {"p50": percentile(0.50), "p95": percentile(0.95), "max": samples[-1]}, + "journalFiles": [str(path) for path in journal_paths], +} +output_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") +if summary["rttMs"]["p95"] < applied_delay: + raise SystemExit( + f"observed IROH RTT p95 {summary['rttMs']['p95']}ms did not reflect applied {applied_delay}ms delay" + ) +print(json.dumps(summary, sort_keys=True)) diff --git a/scripts/run-iroh-release-gate.sh b/scripts/run-iroh-release-gate.sh index 4826b44897fd..95c0f519736c 100755 --- a/scripts/run-iroh-release-gate.sh +++ b/scripts/run-iroh-release-gate.sh @@ -81,6 +81,10 @@ if [[ "$REAL_USAGE" -eq 1 && -z "$REPORT_OUTPUT" ]]; then echo "error: --real-usage requires --report-output" >&2 exit 2 fi +if [[ "$MODE" == relay-only && "$SOAK_PROFILE" == stress && -z "$REPORT_OUTPUT" ]]; then + echo "error: relay-only stress requires --report-output so latency evidence is retained" >&2 + exit 2 +fi [[ "$PHASE_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]] || { echo "error: CMUX_IROH_RELEASE_GATE_PHASE_TIMEOUT_SECONDS must be a positive integer" >&2 exit 2 @@ -379,6 +383,7 @@ PROD_CREDENTIALS_FILE="" PROD_ACCOUNT_STATE_FILE="" PROD_RECOVERY_FILE="" VERCEL_DIR="" +LATENCY_STATE_FILE="" shutdown_prior_gate_simulators() { local simulator_name="$1" @@ -453,6 +458,12 @@ cleanup() { local cleanup_code=0 trap - EXIT INT TERM set +e + if [[ -n "$LATENCY_STATE_FILE" ]]; then + if ! "$SCRIPT_DIR/e2e/iroh-latency-impairment.sh" stop "$LATENCY_STATE_FILE" >/dev/null 2>&1; then + echo "error: latency impairment cleanup failed; the gate cannot pass with network rules still uncertain" >&2 + cleanup_code=1 + fi + fi if [[ -n "$REPORT_WAITER_PID" ]]; then kill "$REPORT_WAITER_PID" >/dev/null 2>&1 || true fi @@ -501,9 +512,10 @@ cleanup() { --credentials-file "$PROD_CREDENTIALS_FILE" \ --api-base-url "$STAGING_BASE_URL" \ --recovery-file "$PROD_RECOVERY_FILE" >/dev/null - cleanup_code=$? - if [[ "$cleanup_code" -ne 0 ]]; then + account_cleanup_code=$? + if [[ "$account_cleanup_code" -ne 0 ]]; then echo "error: production account cleanup gate failed; redacted report: $PROD_RECOVERY_FILE" >&2 + cleanup_code=1 exit_code=1 fi fi @@ -547,6 +559,9 @@ cleanup() { rm -rf "$STATE_DIR" fi fi + if [[ "$cleanup_code" -ne 0 ]]; then + exit_code=1 + fi exit "$exit_code" } @@ -840,6 +855,11 @@ xcrun simctl spawn "$SIMULATOR_ID" defaults write \ xcrun simctl spawn "$SIMULATOR_ID" defaults write \ "$IOS_BUNDLE_ID" cmux.debug.latency-trace -bool true +if [[ "$RAW_MODE" == relayOnly && "$SOAK_PROFILE" == stress ]]; then + LATENCY_STATE_FILE="${REPORT_OUTPUT%.json}-latency.state" + "$SCRIPT_DIR/e2e/iroh-latency-impairment.sh" start "$LATENCY_STATE_FILE" +fi + # The driver owns this unique tag, so restart it unconditionally. A live pairing # socket can otherwise make `cmux_attach_ensure_mac` return without relaunching, # leaving a prior run's transport mode active. @@ -1146,8 +1166,9 @@ if [[ "$REAL_USAGE" -eq 1 ]]; then echo "==> starting real Codex workload in three Mac workspaces" CMUX_E2E_TAG="$TAG" \ CMUX_CODEX_EVIDENCE_DIR="$REAL_USAGE_DIR" \ - CMUX_CODEX_MODEL="${CMUX_CODEX_MODEL:-gpt-5.5-mini}" \ - CMUX_CODEX_DURATION_SECONDS="${CMUX_CODEX_DURATION_SECONDS:-900}" \ + CMUX_CODEX_MODEL="${CMUX_CODEX_MODEL:-gpt-5.3-codex-spark}" \ + CMUX_CODEX_DURATION_SECONDS="${CMUX_CODEX_DURATION_SECONDS:-3600}" \ + CMUX_CODEX_STRICT_MODEL="${CMUX_CODEX_STRICT_MODEL:-1}" \ CMUX_CODEX_SHUTDOWN_FILE="$CODEX_SHUTDOWN_FILE" \ "$SCRIPT_DIR/e2e/iroh-codex-workload.sh" \ > "$REAL_USAGE_DIR/codex-workload.log" 2>&1 & @@ -1199,6 +1220,13 @@ if [[ -n "$REPORT_OUTPUT" ]]; then rm -f "$HOST_DIAGNOSTIC_OUTPUT" echo "warning: Mac Iroh diagnostic capture failed" >&2 fi + if [[ -n "$LATENCY_STATE_FILE" ]]; then + "$SCRIPT_DIR/e2e/summarize-iroh-latency.py" \ + "$LATENCY_STATE_FILE" \ + "${REPORT_OUTPUT%.json}-latency.json" \ + "$(dirname "$REPORT_OUTPUT")" \ + "${REPORT_OUTPUT%.json}" + fi fi REPORT_PATH="$REPORT_PATH" EXPECTED_MODE="$RAW_MODE" EXPECTED_SCENARIO="$GATE_SCENARIO" EXPECTED_SOAK="$SOAK_PROFILE" /usr/bin/python3 <<'PY' @@ -1417,6 +1445,12 @@ if any( for item in cycles ): raise SystemExit("real usage app foreground-to-terminal exceeded two seconds") +if any( + not isinstance(item.get("resume_to_mac_input_seconds"), (int, float)) + or float(item["resume_to_mac_input_seconds"]) > 2.0 + for item in cycles +): + raise SystemExit("real usage resume-to-Mac-input exceeded two seconds") print(json.dumps({"cycles": cycles}, sort_keys=True)) PY_REAL_USAGE fi diff --git a/skills/cmux-cloud-vm/references/commands.md b/skills/cmux-cloud-vm/references/commands.md index 7ad07e5e9cce..f57e806bc404 100644 --- a/skills/cmux-cloud-vm/references/commands.md +++ b/skills/cmux-cloud-vm/references/commands.md @@ -774,6 +774,11 @@ cmux rpc [json-params] # call any v2 method directly, e.g. cmux | `vm.publication_list`, `vm.publication_create`, `vm.publication_verify`, `vm.publication_update`, `vm.publication_delete` | `cloud domains list`, `publish`, `access`, `rm`; `vm.publication_verify` is the app-side publication retry path | | `vm.domain_list`, `vm.domain_verify` | `cloud domains zones`, `cloud domains verify` | | `surface.catalog`, `surface.project`, `surface.new_terminal` | `vm tree` / `surface ls`, `surface open` / `vm open`, `surface new-terminal` / `vm agent` | +| `vm.env_set`, `vm.file_put` | Secret-safe environment transfer and authenticated file upload primitives used by `vm env set` and `vm push` | +| `vm.pause`, `vm.resume` | Suspend or resume a machine without deleting it | +| `vm.reflection` | Provider and transport reflection data used by diagnostics | +| `vm.snapshot_delete`, `vm.snapshot_list` | List and remove snapshots for the selected machine | +| `vm.terminal_output`, `vm.terminal_wait_exit` | Read terminal output incrementally and wait for process exit | The authenticated public-domain workflow is **shipped on this branch**: use `cmux cloud domains` for generated or custom HTTPS publications. `cmux vm open diff --git a/tests/test_iroh_monitor_simulator_plan.py b/tests/test_iroh_monitor_simulator_plan.py index 5beb2383d4db..1f8f2bd8f615 100644 --- a/tests/test_iroh_monitor_simulator_plan.py +++ b/tests/test_iroh_monitor_simulator_plan.py @@ -15,7 +15,7 @@ def test_workflow_bounds_the_gate_step(self): if step.get("name") == "Run Iroh gate" ) self.assertIn("timeout-minutes", step) - self.assertIn("75", step["timeout-minutes"]) + self.assertIn("125", step["timeout-minutes"]) self.assertIn("25", step["timeout-minutes"]) def test_gate_script_has_phase_timeout_and_reason(self):