Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,22 @@ func TestGoldenMigrationPreparationRequiresCompatibleBootstrapAndStableBackendHe
"max_sample_gap_ms": 5_000,
"backend_membership_sha256": strings.Repeat("d", 64),
}
result["run"].(map[string]any)["instance"] = "subrouter-staging"
routing := result["routing"].(map[string]any)
routing["active_matcher"] = "staging-subrouter"
routing["canary"] = map[string]any{
"host": "front-canary.staging.sr.cmux.internal",
"matcher": "staging-subrouter-front-canary",
"backend_url": routing["front_backend_url"],
"map_updated_at": "2026-08-01T23:59:00Z",
"first_observed_at": "2026-08-02T00:00:00Z",
"verified_at": "2026-08-02T00:05:00Z",
"stable_duration_ms": 300_000,
"first_proof_attempts": 1,
"verified_proof_attempts": 1,
"first_session_sha256": strings.Repeat("e", 64),
"verified_session_sha256": strings.Repeat("f", 64),
}
return result
}
validateGo := func(value map[string]any) error {
Expand Down Expand Up @@ -122,13 +138,45 @@ func TestGoldenMigrationPreparationRequiresCompatibleBootstrapAndStableBackendHe
fractionalHealth := fractionalMillisecond["front"].(map[string]any)["backend_health"].(map[string]any)
fractionalHealth["verified_at"] = "2026-08-02T00:05:00.001Z"
fractionalHealth["duration_ms"] = 300_001
fractionalCanary := fractionalMillisecond["routing"].(map[string]any)["canary"].(map[string]any)
fractionalCanary["verified_at"] = "2026-08-02T00:05:00.001Z"
fractionalCanary["stable_duration_ms"] = 300_001
if err := validateGo(fractionalMillisecond); err != nil {
t.Fatalf("Go validator rejected exact millisecond readiness duration: %v", err)
}
if err := validatePython(fractionalMillisecond); err != nil {
t.Fatalf("Python validator rejected exact millisecond readiness duration: %v", err)
}

missingCanary := clone(valid)
delete(missingCanary["routing"].(map[string]any), "canary")
if err := validateGo(missingCanary); err == nil {
t.Fatal("Go validator accepted preparation without a front canary proof")
}
if err := validatePython(missingCanary); err == nil {
t.Fatal("Python validator accepted preparation without a front canary proof")
}

shortCanary := clone(valid)
shortCanaryProof := shortCanary["routing"].(map[string]any)["canary"].(map[string]any)
shortCanaryProof["verified_at"] = "2026-08-02T00:04:59.999Z"
shortCanaryProof["stable_duration_ms"] = 299_999
if err := validateGo(shortCanary); err == nil {
t.Fatal("Go validator accepted a sub-five-minute front canary proof")
}
if err := validatePython(shortCanary); err == nil {
t.Fatal("Python validator accepted a sub-five-minute front canary proof")
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

wrongCanaryHost := clone(valid)
wrongCanaryHost["routing"].(map[string]any)["canary"].(map[string]any)["host"] = "attacker.invalid"
if err := validateGo(wrongCanaryHost); err == nil {
t.Fatal("Go validator accepted the wrong front canary host")
}
if err := validatePython(wrongCanaryHost); err == nil {
t.Fatal("Python validator accepted the wrong front canary host")
}

missing := clone(valid)
delete(missing["front"].(map[string]any), "backend_health")
if err := validateGo(missing); err == nil {
Expand Down Expand Up @@ -639,7 +687,7 @@ func validGoldenRetirementEvidence(mode string) *goldenDeployEvidence {
func validGoldenMigrationBaseEvidence(evidenceType, mode string) *goldenMigrationEvidence {
return &goldenMigrationEvidence{
Schema: goldenDeployEvidenceSchema, EvidenceType: evidenceType, Mode: mode, Success: true,
Run: goldenDeployRun{ID: "golden-run", Project: "project", Zone: "zone", Instance: "instance"},
Run: goldenDeployRun{ID: "golden-run", Project: "project", Zone: "zone", Instance: "subrouter-staging"},
Release: goldenDeployRelease{
Tag: goldenPinnedCandidateTag, SHA256: strings.Repeat("b", 64), SourceRevision: strings.Repeat("c", 40),
TagOnMain: true, AttestationVerified: true, Immutable: true,
Expand Down Expand Up @@ -674,7 +722,14 @@ func validGoldenMigrationPreparationEvidence() *goldenMigrationEvidence {
evidence.Routing = goldenMigrationRouting{
URLMap: "url-map", LegacyBackend: "legacy-backend", FrontBackend: "front-backend",
LegacyBackendURL: "https://example.test/legacy", FrontBackendURL: "https://example.test/front",
Current: "legacy",
ActiveMatcher: "staging-subrouter", Current: "legacy",
Canary: goldenMigrationCanary{
Host: "front-canary.staging.sr.cmux.internal", Matcher: "staging-subrouter-front-canary",
BackendURL: "https://example.test/front", MapUpdatedAt: "2026-08-01T23:59:00Z",
FirstObservedAt: "2026-08-02T00:00:00Z", VerifiedAt: "2026-08-02T00:05:00Z",
StableDurationMillis: 300_000, FirstProofAttempts: 1, VerifiedProofAttempts: 1,
FirstSessionSHA256: strings.Repeat("e", 64), VerifiedSessionSHA256: strings.Repeat("f", 64),
},
}
evidence.EvidenceEmittedAt = "2026-08-02T00:05:01Z"
return evidence
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ esac
t.Setenv("SUBROUTER_GOLDEN_FAKE_PROCESS_STATE", processState)
t.Setenv("SUBROUTER_GOLDEN_FAKE_PROCESS_PARENT_OWNED", "1")
t.Setenv("SUBROUTER_GOLDEN_FAKE_MIGRATION_RETRY_ONCE", "1")
t.Setenv("SUBROUTER_GOLDEN_FAKE_REQUIRE_SESSIONS_DURING_PREPARE", "1")
t.Setenv("PATH", fakeBin+string(os.PathListSeparator)+os.Getenv("PATH"))
artifacts := filepath.Join(root, "artifacts")
err = runGolden([]string{
Expand Down
22 changes: 11 additions & 11 deletions cmd/subrouter-transport-observer/golden_migration_cycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,6 @@ type goldenMigrationResult struct {
func (r *goldenRunner) runMigrationCycle(ctx context.Context, inputs goldenCycleInputs) (goldenMigrationResult, error) {
var result goldenMigrationResult
var err error
result.preparation, err = r.runMigrationEvidenceAction(ctx, goldenMigrationActionOptions{
label: "migration-preparation", argv: r.options.migrationPrepare,
expect: "front-migration-preparation", evidenceName: "migration-preparation.json",
})
if err != nil {
return result, err
}
if err := r.validateGoldenCandidateIdentity(result.preparation.migrationCanonical); err != nil {
return result, err
}

result.initial, err = r.startCycleInitialSessions(ctx, inputs, false)
if err != nil {
return result, err
Expand All @@ -51,6 +40,17 @@ func (r *goldenRunner) runMigrationCycle(ctx context.Context, inputs goldenCycle
}
}()

result.preparation, err = r.runMigrationEvidenceAction(ctx, goldenMigrationActionOptions{
label: "migration-preparation", argv: r.options.migrationPrepare,
expect: "front-migration-preparation", evidenceName: "migration-preparation.json",
})
if err != nil {
return result, err
}
if err := r.validateGoldenCandidateIdentity(result.preparation.migrationCanonical); err != nil {
return result, err
}

var frontProof *goldenSession
result.rehearsalCutover, frontProof, err = r.runMigrationTransitionWithProof(
ctx, "rehearsal-cutover", "migration-candidate-front-rehearsal", result.preparation,
Expand Down
83 changes: 69 additions & 14 deletions cmd/subrouter-transport-observer/golden_migration_evidence.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,19 +57,35 @@ type goldenMigrationPredecessor struct {
}

type goldenMigrationRouting struct {
URLMap string `json:"url_map"`
LegacyBackend string `json:"legacy_backend"`
FrontBackend string `json:"front_backend"`
LegacyBackendURL string `json:"legacy_backend_url"`
FrontBackendURL string `json:"front_backend_url"`
Current string `json:"current"`
Before string `json:"before"`
After string `json:"after"`
SourceBackendURL string `json:"source_backend_url"`
DestinationBackendURL string `json:"destination_backend_url"`
Active string `json:"active"`
LegacyBackendRetained bool `json:"legacy_backend_retained"`
AcceptingNewPublic bool `json:"accepting_new_public"`
URLMap string `json:"url_map"`
ActiveMatcher string `json:"active_matcher"`
LegacyBackend string `json:"legacy_backend"`
FrontBackend string `json:"front_backend"`
LegacyBackendURL string `json:"legacy_backend_url"`
FrontBackendURL string `json:"front_backend_url"`
Current string `json:"current"`
Before string `json:"before"`
After string `json:"after"`
SourceBackendURL string `json:"source_backend_url"`
DestinationBackendURL string `json:"destination_backend_url"`
Active string `json:"active"`
LegacyBackendRetained bool `json:"legacy_backend_retained"`
AcceptingNewPublic bool `json:"accepting_new_public"`
Canary goldenMigrationCanary `json:"canary"`
}

type goldenMigrationCanary struct {
Host string `json:"host"`
Matcher string `json:"matcher"`
BackendURL string `json:"backend_url"`
MapUpdatedAt string `json:"map_updated_at"`
FirstObservedAt string `json:"first_observed_at"`
VerifiedAt string `json:"verified_at"`
StableDurationMillis int64 `json:"stable_duration_ms"`
FirstProofAttempts int64 `json:"first_proof_attempts"`
VerifiedProofAttempts int64 `json:"verified_proof_attempts"`
FirstSessionSHA256 string `json:"first_session_sha256"`
VerifiedSessionSHA256 string `json:"verified_session_sha256"`
}

type goldenMigrationLegacy struct {
Expand Down Expand Up @@ -241,8 +257,22 @@ func validateGoldenMigrationEvidence(evidence *goldenMigrationEvidence, expected
}
switch expected {
case "front-migration-preparation":
expectedMatcher, expectedCanaryMatcher, expectedCanaryHost := "", "", ""
switch evidence.Run.Instance {
case "subrouter-staging":
expectedMatcher = "staging-subrouter"
expectedCanaryMatcher = "staging-subrouter-front-canary"
expectedCanaryHost = "front-canary.staging.sr.cmux.internal"
case "subrouter-team":
expectedMatcher = "__root__"
expectedCanaryMatcher = "subrouter-front-canary"
expectedCanaryHost = "front-canary.sr.cmux.internal"
default:
return failGolden("migration_preparation_target_invalid")
}
if evidence.Mode != "prepare" || evidence.Routing.Current != "legacy" ||
evidence.Routing.URLMap == "" || evidence.Routing.LegacyBackend == "" || evidence.Routing.FrontBackend == "" ||
evidence.Routing.ActiveMatcher != expectedMatcher ||
evidence.Routing.LegacyBackendURL == evidence.Routing.FrontBackendURL ||
!strings.HasPrefix(evidence.Routing.LegacyBackendURL, "https://") ||
!strings.HasPrefix(evidence.Routing.FrontBackendURL, "https://") ||
Expand All @@ -253,6 +283,26 @@ func validateGoldenMigrationEvidence(evidence *goldenMigrationEvidence, expected
evidence.Front.WorkerChecksum != evidence.Bootstrap.SHA256 || !evidence.Front.Ready {
return failGolden("migration_preparation_invalid")
}
canary := evidence.Routing.Canary
if canary.Host != expectedCanaryHost || canary.Matcher != expectedCanaryMatcher ||
canary.BackendURL != evidence.Routing.FrontBackendURL ||
canary.FirstProofAttempts < 1 || canary.FirstProofAttempts > 600 ||
canary.VerifiedProofAttempts < 1 || canary.VerifiedProofAttempts > 600 ||
!validGoldenSHA256(canary.FirstSessionSHA256) || !validGoldenSHA256(canary.VerifiedSessionSHA256) {
return failGolden("migration_canary_invalid")
}
mapUpdatedAt, err := parseGoldenEvidenceTime(canary.MapUpdatedAt)
if err != nil {
return err
}
firstObservedAt, err := parseGoldenEvidenceTime(canary.FirstObservedAt)
if err != nil {
return err
}
canaryVerifiedAt, err := parseGoldenEvidenceTime(canary.VerifiedAt)
if err != nil {
return err
}
stableSince, err := parseGoldenEvidenceTime(evidence.Front.BackendHealth.StableSince)
if err != nil {
return err
Expand All @@ -274,6 +324,7 @@ func validateGoldenMigrationEvidence(evidence *goldenMigrationEvidence, expected
}
samplesCoverDuration = evidence.Front.BackendHealth.HealthySamples-1 >= requiredIntervals
}
canaryStableDuration := canaryVerifiedAt.Sub(firstObservedAt)
if err != nil || !evidence.Front.BackendHealth.AllHealthy || verifiedAt.Before(stableSince) ||
stableDuration < goldenBackendHealthStabilityLimit ||
stableDuration > 15*time.Minute ||
Expand All @@ -283,7 +334,11 @@ func validateGoldenMigrationEvidence(evidence *goldenMigrationEvidence, expected
evidence.Front.BackendHealth.MaxSampleGapMillis < 0 ||
evidence.Front.BackendHealth.MaxSampleGapMillis > 15_000 ||
!samplesCoverDuration ||
!validGoldenSHA256(evidence.Front.BackendHealth.BackendMembershipSHA256) || emittedAt.Before(verifiedAt) {
!validGoldenSHA256(evidence.Front.BackendHealth.BackendMembershipSHA256) ||
firstObservedAt.Before(mapUpdatedAt) || stableSince.Before(firstObservedAt) ||
canaryVerifiedAt.Before(verifiedAt) || emittedAt.Before(canaryVerifiedAt) ||
canaryStableDuration < goldenBackendHealthStabilityLimit || canaryStableDuration > 20*time.Minute ||
canary.StableDurationMillis != canaryStableDuration.Milliseconds() {
return failGolden("migration_backend_health_invalid")
}
return nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ func fakeAction(args []string) {
os.Exit(9)
}
operation := args[0]
if operation == "migration-prepare" && os.Getenv("SUBROUTER_GOLDEN_FAKE_REQUIRE_SESSIONS_DURING_PREPARE") == "1" {
entries, err := os.ReadDir(os.Getenv("SUBROUTER_GOLDEN_FAKE_PROCESS_STATE"))
// The harness process and this action are already registered. Require at
// least one separately held Codex session before migration preparation.
if err != nil || len(entries) <= 2 {
os.Exit(9)
}
}
if logPath := os.Getenv("ACTION_LOG"); logPath != "" {
file, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
Expand Down Expand Up @@ -492,11 +500,19 @@ func fakeMigrationPreparation(candidate, revision string) map[string]any {
stableSince := verifiedAt.Add(-5 * time.Minute)
return map[string]any{
"schema": "subrouter.gcp.deploy-evidence/v1", "evidence_type": "front-migration-preparation", "mode": "prepare", "success": true,
"run": map[string]any{"id": "golden-migration-prepare", "project": "test-project", "zone": "test-zone", "instance": "test-instance"},
"run": map[string]any{"id": "golden-migration-prepare", "project": "test-project", "zone": "test-zone", "instance": "subrouter-staging"},
"release": fakeMigrationRelease(candidate, revision), "bootstrap": fakeMigrationBootstrap(), "predecessor": fakeMigrationPredecessor(),
"routing": map[string]any{
"url_map": "test-map", "legacy_backend": "legacy-backend", "front_backend": "front-backend",
"legacy_backend_url": "https://legacy.test", "front_backend_url": "https://front.test", "current": "legacy",
"active_matcher": "staging-subrouter",
"canary": map[string]any{
"host": "front-canary.staging.sr.cmux.internal", "matcher": "staging-subrouter-front-canary",
"backend_url": "https://front.test", "map_updated_at": stableSince.Add(-time.Minute).Format(time.RFC3339Nano),
"first_observed_at": stableSince.Format(time.RFC3339Nano), "verified_at": verifiedAt.Format(time.RFC3339Nano),
"stable_duration_ms": 300_000, "first_proof_attempts": 1, "verified_proof_attempts": 1,
"first_session_sha256": strings.Repeat("e", 64), "verified_session_sha256": strings.Repeat("f", 64),
},
},
"legacy": map[string]any{
"service": "subrouter.service", "generation": "legacy-generation", "checksum": "99fcd10d912184c160370eb228b382795101f2b5b2467244f995aa2d10b0c323", "accepting_new_public": true,
Expand Down
Loading