diff --git a/dag/extdeps/bmc/phosphor_fan_control.dag b/dag/extdeps/bmc/phosphor_fan_control.dag index 323d27b9bab..57f7806a39e 100644 --- a/dag/extdeps/bmc/phosphor_fan_control.dag +++ b/dag/extdeps/bmc/phosphor_fan_control.dag @@ -123,4 +123,4 @@ fn phosphor_fan_failsafe_condition_label(condition: PhosphorFanFailsafeCondition // the fan-policy lane with a policy module that supplies its family authority and converge route. // Adding an OpenBMC host to the fleet does not retire this row; adding one that reaches this // standing does. -data phosphor_fan_control_first_consumer_frontier: DissolutionCondition = unbound_dissolution(description: "dissolves when an OpenBMC-family host's policy layer supplies a product.fan_policy_standing ConvergeRouteBound whose authority is phosphor_fan_zone_policy_contract, as gunbc.machine_intake_mtcollins1_fan_observe supplies the MegaRAC authority today; until then this module's only executing consumer is test.claim.fan_policy_standing_witness. An OpenBMC host merely joining the fleet does not retire this row - one reaching this standing does.") +data phosphor_fan_control_first_consumer_frontier: DissolutionCondition = unbound_dissolution(description: "dissolves when an OpenBMC-family host's policy layer supplies a product.fan_policy_standing ConvergeRouteBound whose authority is phosphor_fan_zone_policy_contract, as gunbc.machine_intake_fan_observe supplies the MegaRAC authority today via gunbc.fleet_fan_converge_routes fan_observation_authority_for_family; until then this module's only executing consumer is test.claim.fan_policy_standing_witness. An OpenBMC host merely joining the fleet does not retire this row - one reaching this standing does.") diff --git a/dag/gunbc/fleet/fleet_fan_converge_routes.dag b/dag/gunbc/fleet/fleet_fan_converge_routes.dag index 3e8e9b8574e..ec02266fdb4 100644 --- a/dag/gunbc/fleet/fleet_fan_converge_routes.dag +++ b/dag/gunbc/fleet/fleet_fan_converge_routes.dag @@ -56,3 +56,27 @@ fn fan_converge_route_for_family(family: BmcFirmwareFamily) -> FanDutyConvergeRo AmiMegaRac => megarac_fan_converge_route } } + +// THE AUTHORITY AN OBSERVED FAN SERIES IS READ AGAINST, PER BMC FAMILY (managed-host cut 5). Which +// family's policy governs a host's fans is selected from what its controller was OBSERVED to be, never +// assumed, so this dispatch takes an observed family and has no arm for "unobserved": a host whose +// family is unread refuses before it reaches here (gunbc.machine_intake_fan_observe). It is the same +// peripheral realization as the route dispatch above, and for the same reason it is not in either +// extdeps module or in product.fan_policy_standing. The two rows differ from the routes' on purpose: +// an observation is read against the family's POLICY (what holds a failsafe and when), while a converge +// route names the WRITE surface; OpenBMC's phosphor zone contract is both, MegaRAC's policy and its +// unbound fan-mode surface are two declarations. +data megarac_fan_observation_authority: FanPolicyAuthority = FanPolicyAuthority { + declaration: decl_ref("extdeps.bmc.megarac_fan_policy", "megarac_fan_policy_standing"), +} + +data openbmc_fan_observation_authority: FanPolicyAuthority = FanPolicyAuthority { + declaration: decl_ref("extdeps.bmc.phosphor_fan_control", "phosphor_fan_zone_policy_contract"), +} + +fn fan_observation_authority_for_family(family: BmcFirmwareFamily) -> FanPolicyAuthority { + match family { + OpenBmc => openbmc_fan_observation_authority + AmiMegaRac => megarac_fan_observation_authority + } +} diff --git a/dag/gunbc/host/managed_host.dag b/dag/gunbc/host/managed_host.dag index a810c7e4720..ba2528a2fae 100644 --- a/dag/gunbc/host/managed_host.dag +++ b/dag/gunbc/host/managed_host.dag @@ -577,6 +577,8 @@ fn managed_host_fixture( decl_ref(module_path: "test.claim.megarac_managed_host_witness", decl_name: "w_an_account_is_released_only_with_the_controller_it_was_bound_to"), decl_ref(module_path: "test.claim.machine_intake.dimm_physical_orientation_witness", decl_name: "w_the_same_anchors_on_another_reported_board_are_refused_unmodeled"), decl_ref(module_path: "test.claim.machine_intake.dimm_physical_orientation_witness", decl_name: "w_an_unobserved_controller_is_refused_at_the_binding_not_as_an_unmodeled_board"), + decl_ref(module_path: "test.claim.machine_intake.fan_observe_witness_test", decl_name: "an_openbmc_bound_row_is_read_against_the_phosphor_policy"), + decl_ref(module_path: "test.claim.machine_intake.fan_observe_witness_test", decl_name: "a_row_whose_family_is_unobserved_refuses_naming_the_missing_observation"), ] { ManagedHost { host: host, access: access, capability_row: capability_row, secure: secure, unit_hold: unit_hold, smpro: smpro } diff --git a/dag/gunbc/host/managed_host_unit_hold.dag b/dag/gunbc/host/managed_host_unit_hold.dag index 553c8dcc722..9c53c47f790 100644 --- a/dag/gunbc/host/managed_host_unit_hold.dag +++ b/dag/gunbc/host/managed_host_unit_hold.dag @@ -593,7 +593,7 @@ fn unit_hold_not_admitted(admission: UnitHoldHostAdmission) -> UnitHoldAcquisiti // nobody accounted for. Holding the unit makes the probe, the boot and maintenance exclude each other. fn kvm_observer_acquire_unit_hold(host: HostIdentity, run_id: NonEmptyStr) -> UnitHoldAcquisition admit_callers: [ - decl_ref(module_path: "gunbc.machine_intake_mtcollins1_kvm_observer_observe", decl_name: "mtcollins1_kvm_observer_observe_wet"), + decl_ref(module_path: "gunbc.machine_intake_megarac_kvm_observer_observe", decl_name: "megarac_kvm_observer_observe"), ] { match unit_hold_admission_for(host: host as NonEmptyStr) { @@ -685,7 +685,7 @@ fn probe_hold_release_outcome_text(o: ProbeHoldReleaseOutcome) -> String { fn kvm_observer_release_unit_hold(host: HostIdentity, run_id: NonEmptyStr) -> ProbeHoldReleaseOutcome admit_callers: [ - decl_ref(module_path: "gunbc.machine_intake_mtcollins1_kvm_observer_observe", decl_name: "mtcollins1_kvm_observer_release_wet"), + decl_ref(module_path: "gunbc.machine_intake_megarac_kvm_observer_observe", decl_name: "megarac_kvm_observer_release"), ] { match unit_hold_admission_for(host: host as NonEmptyStr) { diff --git a/dag/gunbc/machine_intake/fan_observe.dag b/dag/gunbc/machine_intake/fan_observe.dag new file mode 100644 index 00000000000..abd0fb99f52 --- /dev/null +++ b/dag/gunbc/machine_intake/fan_observe.dag @@ -0,0 +1,248 @@ +module gunbc.machine_intake_fan_observe + +import std.types { Int, List, NonEmptyStr, String } +import std.decl_ref { decl_ref } +import std.measure { ObservationRoundCount, Second, percent, second_count } +import std.process { ExitSuccess, ProcessExit, exit_failure } +import std.optional { Present, Absent } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.bmc.endpoint { BmcControllerEndpoint } +import extdeps.bmc.access_profile { BmcAccessObservation } +import product.host_identity { HostIdentity } +import product.fan_policy_standing { + FanDutyCeilingVerdict, + ExcursionBound, + FanDutyExcursionOverCeiling, + FanDutyNotReported, + NoPostTransitionWindow, + OutlivedPostTransitionWindow, + WithinPostTransitionWindow, + FanDutySteadyCurveOverCeiling, + FanDutyWithinCeiling, + FanPolicyAuthority, + FanPolicySample, + fan_policy_standing_label, + fan_policy_standing_of_sample, +} +import gunbc.machine_intake_bmc_secure { BmcSecureRefusalCause } +import gunbc.managed_host { + ManagedHostLookup, ManagedHostFound, ManagedHostFoundUnderDeclaredDrop, ManagedHostNotManaged, + ManagedHostBindingStanding, ManagedHostBound, ManagedHostAccessUnobserved, ManagedHostNotSecured, ManagedHostEndpointMismatch, +} +import gunbc.fleet_fan_converge_routes { fan_observation_authority_for_family } +import gunbc.machine_intake_bmc_fan_observation { + FanObservationRefused, + FanObservationRound, + FanObservationSeries, + FanObservationTaken, + elapsed_at_round, + first_posted_round_index, + observe_fan_policy, + render_series, + samples_of_round, +} +import gunbc.fleet_acoustic_intent { residence_duty_verdict } + +// ONE MANAGED HOST'S FAN OBSERVATION, OVER ITS BINDING (managed-host cut 5; first written for Mt. +// Collins). The producer (gunbc.machine_intake_bmc_fan_observation) reads the sensor series over IPMI, +// which no BMC family owns; what is family-specific is only the POLICY the series is read against, and +// that is SELECTED from the family the host's controller was observed to be +// (gunbc.fleet_fan_converge_routes fan_observation_authority_for_family), never assumed. A host whose +// controller was never observed has no family to select from and REFUSES with a cause naming the +// missing observation: it is never read against either policy by default. That is Mt. Jade's case +// (gunbc.machine_intake_mtjade1_access_observation: its family is unobserved). +// +// The controller, its account and the host the samples are attributed to all come off the binding. +// How many samples, how far apart, the transition window and where the receipt lands are the caller's +// policy, so a second host needs a second binding and no edit here. + +// THE BOUND HOST WITH ITS SELECTED AUTHORITY. Sealed: minted only by fan_observe_from_standing, which +// reads it off a ManagedHostBinding (itself sole-constructed by gunbc.managed_host), so no controller, +// account or authority reaches the read except through a binding. +type FanObserveHost sole_constructor { + host: HostIdentity + observation: BmcAccessObservation + account: NonEmptyStr + authority: FanPolicyAuthority +} + +type FanObserveRefusal + = FanHostNotManaged { host: NonEmptyStr } + | FanHostFamilyUnobserved { endpoint: BmcControllerEndpoint } + | FanHostNotSecured { cause: BmcSecureRefusalCause } + | FanHostEndpointMismatch { observed: BmcControllerEndpoint, secured: BmcControllerEndpoint } + +type FanObserveAdmission + = FanObserveAdmitted { host: FanObserveHost } + | FanObserveRefused { cause: FanObserveRefusal } + +fn fan_observe_from_standing(standing: ManagedHostBindingStanding) -> FanObserveAdmission { + match standing { + ManagedHostBound { binding: b } => + FanObserveAdmitted { + host: FanObserveHost { + host: b.host.host, + observation: b.observation, + account: b.account, + authority: fan_observation_authority_for_family(family: b.observation.identity.firmware.family), + }, + } + ManagedHostAccessUnobserved { endpoint: e } => FanObserveRefused { cause: FanHostFamilyUnobserved { endpoint: e } } + ManagedHostNotSecured { cause: c } => FanObserveRefused { cause: FanHostNotSecured { cause: c } } + ManagedHostEndpointMismatch { observed: o, secured: sec } => FanObserveRefused { cause: FanHostEndpointMismatch { observed: o, secured: sec } } + } +} + +fn fan_observe_admission(lookup: ManagedHostLookup) -> FanObserveAdmission { + match lookup { + ManagedHostNotManaged { host: h } => FanObserveRefused { cause: FanHostNotManaged { host: h } } + ManagedHostFound { standing: s } => fan_observe_from_standing(standing: s) + ManagedHostFoundUnderDeclaredDrop { standing: s, drop: _, standing_subject: _, current_subject: _ } => + fan_observe_from_standing(standing: s) + } +} + +fn fan_observe_refusal_text(cause: FanObserveRefusal) -> NonEmptyStr { + let why = match cause { + FanHostNotManaged { host: h } => join(["'", h as String, "' is not a managed host (gunbc.managed_host managed_hosts has no row for it)"], "") + FanHostFamilyUnobserved { endpoint: e } => join(["the controller at ", e.host as String, " was never observed, so its BMC family -- and so the fan policy its series is read against -- is unread"], "") + FanHostNotSecured { cause: _ } => "the controller is not BmcSecured, so this run holds no account to read sensors with" + FanHostEndpointMismatch { observed: o, secured: sec } => join(["the observed controller ", o.host as String, " is not the secured one ", sec.host as String], "") + } + concat(why, "; contacting NO host") as NonEmptyStr +} + +// THE AUTHORITY AN ADMITTED HOST'S SERIES IS READ AGAINST, so a caller can name it without the carrier. +fn fan_observe_authority(host: FanObserveHost) -> FanPolicyAuthority { + host.authority +} + +// THE CALLER'S POLICY FOR ONE SERIES: its length, its spacing, the post-transition window its standings +// are judged in, and where its receipt lands. +type FanObservePolicy { + rounds: ObservationRoundCount + interval: Second + transition_window: Second + receipt_path: String +} + +// THE COMPARISON DUTY IS ONE HUNDRED PERCENT, AND THAT IS THIS RUN'S QUESTION RATHER THAN A FACT +// ABOUT ANY FAMILY. Neither family authority publishes a failsafe duty and this module may not invent +// one. What passing 100 asks is narrower and defensible - "is the drive at maximum" - which is exactly +// the state the operator described on Mt. Jade. It is spelled as a literal because percent() admits a +// literal only (std.measure percent_from_computed_int_frontier). +// +// NINETY-FIVE PERCENT IS THIS LANE'S STATED TOLERANCE, NOT A FACT ABOUT ANY FAN. It selects between +// NearCatalogMaximum and BelowCatalogMaximum on a SPEED observation; it cannot reach a failsafe verdict, +// because no tolerance converts RPM into a commanded duty (operator ruling 2026-09-18). No fan part is +// modeled for any managed host, so catalog_maximum is Absent and the tolerance is not consulted today. +// +// ONE STANDING PER SAMPLE, EACH WITH THE SITE DUTY VERDICT BESIDE IT, so the verdict can be re-derived +// from the artifact rather than from a summary. +fn render_sample_standing(host: FanObserveHost, window: Second, sample: FanPolicySample, elapsed: Second?) -> String { + let standing = fan_policy_standing_of_sample( + host: host.host, + authority: host.authority, + sample: sample, + failsafe_duty: percent(100), + catalog_maximum: Absent, + at_maximum_tolerance: percent(95), + elapsed_since_post: elapsed, + transition_window: window, + ) + join([ + " standing fan=", sample.endpoint.name as String, + " at=", sample.observed_at as String, + " verdict=", fan_policy_standing_label(standing: standing) as String, + " residence_duty=", render_duty_verdict(verdict: residence_duty_verdict(standing: standing)), + ], "") +} + +// THE TWO OVER-ARMS RENDER DISTINCTLY, because their remedies differ: a curve over the ceiling is +// something to converge, and an excursion over it is something to bound in time or to end by +// powering the machine off. +fn render_duty_verdict(verdict: FanDutyCeilingVerdict) -> String { + match verdict { + FanDutyWithinCeiling { duty: _, ceiling: _ } => "WithinCeiling" + FanDutySteadyCurveOverCeiling { duty: _, ceiling: _ } => "SteadyCurveOverCeiling" + FanDutyExcursionOverCeiling { duty: _, ceiling: _, bound: b } => join(["ExcursionOverCeiling(", render_excursion_bound(bound: b), ")"], "") + FanDutyNotReported { cause: _ } => "NotReported" + } +} + +// THE BOUND RENDERS AS WHAT IT IS, and the pre-POST case says so in words rather than as a zero: nothing +// bounds such an excursion but powering the machine off. +fn render_excursion_bound(bound: ExcursionBound) -> String { + match bound { + NoPostTransitionWindow => "no post-transition window; bounded only by power-off" + WithinPostTransitionWindow { elapsed: e, window: w } => + join(["within window, elapsed=", to_string(second_count(s: e)), "s of ", to_string(second_count(s: w)), "s"], "") + OutlivedPostTransitionWindow { elapsed: e, window: w } => + join(["OUTLIVED window, elapsed=", to_string(second_count(s: e)), "s past ", to_string(second_count(s: w)), "s"], "") + } +} + +// EACH ROUND'S SAMPLES ARE STAMPED WITH THAT ROUND'S OWN ELAPSED, never with the series'. The first +// posted index is a property of the series and is computed once; the elapsed is the round's. +fn render_observation(host: FanObserveHost, window: Second, rounds: List, interval: Second) -> String { + let first_posted = first_posted_round_index(rounds: rounds) + join([ + render_series(rounds: rounds), + join( + map(rounds, r => render_round_standings(host: host, window: window, round: r, first_posted: first_posted, interval: interval)), + "\n", + ), + ], "\n") +} + +fn render_round_standings(host: FanObserveHost, window: Second, round: FanObservationRound, first_posted: Int?, interval: Second) -> String { + let elapsed = elapsed_at_round(round: round, first_posted_index: first_posted, interval: interval) + join( + map( + samples_of_round(host: host.host, round: round), + s => render_sample_standing(host: host, window: window, sample: s, elapsed: elapsed), + ), + "\n", + ) +} + +// THE ADMITTED READ. Sealed to the one entry that binds a managed host's row to it; it returns a +// ProcessExit, never a carrier. The credential file is the caller's +// (gunbc.machine_intake_mtcollins1_bmc_secure_observation mtcollins1_managed_secret_fetch_frontier). +fn fan_observe_with(host: FanObserveHost, password_file: NonEmptyStr, policy: FanObservePolicy) -> ProcessExit + admit_callers: [ + decl_ref(module_path: "gunbc.machine_intake_mtcollins1_fan_observe", decl_name: "mtcollins1_fan_observe_wet"), + ] +{ + fan_write_observation( + host: host, + policy: policy, + series: observe_fan_policy( + bmc_host: host.observation.endpoint.host, + username: host.account, + password_file: password_file, + rounds: policy.rounds, + interval: policy.interval, + ), + ) +} + +// THE RECEIPT IS WRITTEN ONLY FOR A SERIES THAT WAS TAKEN. A refused observation's text would leave an +// artifact that LOOKS like a measurement attached to a run that failed. +fn fan_write_observation(host: FanObserveHost, policy: FanObservePolicy, series: FanObservationSeries) -> ProcessExit { + match series { + FanObservationRefused { cause: cause } => + exit_failure(reason: concat("fan observation: ", cause as String)) + FanObservationTaken { rounds: rounds, interval: interval } => { + let written = Filesystem.Write( + path: policy.receipt_path, + content: render_observation(host: host, window: policy.transition_window, rounds: rounds, interval: interval), + ) + if written.success { + ExitSuccess + } else { + exit_failure(reason: concat("fan observation: the series was taken and its receipt could not be written: ", written.error)) + } + } + } +} diff --git a/dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag b/dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag new file mode 100644 index 00000000000..7e57c27538d --- /dev/null +++ b/dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag @@ -0,0 +1,275 @@ +module gunbc.machine_intake_megarac_kvm_observer_observe + +import std.types { Bool, Int, List, NonEmptyStr, String } +import std.decl_ref { decl_ref } +import std.measure { Second } +import std.process { ExitSuccess, ExitFailure, ProcessExit, exit_failure } +import std.optional { Present, Absent } +import extdeps.filesystem.filesystem_io { Filesystem } +import product.host_identity { HostIdentity } +import gunbc.runner_browser_toolchain { runner_browser_toolchain_here_wet } +import gunbc.owned_process { OwnedRelease, OwnedReleased, OwnedAlreadyGone, OwnedReleaseNotOwned, OwnedNotRecorded, OwnedForeign, owned_process_release, owned_release_text } +import gunbc.megarac_managed_host { AdmittedMegaRacHost, admitted_megarac_host_identity } +import gunbc.managed_host_unit_hold { + UnitHeld, UnitHoldRefused, kvm_observer_acquire_unit_hold, unit_hold_release, + ProbeHoldReleased, ProbeHoldAlreadyFree, ProbeHoldNotOurs, ProbeHoldStale, ProbeHoldUnreleased, + kvm_observer_release_unit_hold, probe_hold_release_outcome_text, +} +import gunbc.machine_intake_megarac_kvm_still { + KvmObserverRecord, KvmObserverStart, KvmObserverStarted, KvmObserverNotStarted, + KvmHandoffAdmitted, KvmHandoffRefused, kvm_handoff_admission, + KvmStillHashed, KvmTeardownRecorded, KvmTeardownNotNeeded, KvmTerminalStillAnswered, + KvmSessionRelease, KvmSessionReleased, KvmSessionNoSession, KvmBrowserClose, KvmBrowserClosed, KvmBrowserNotLaunched, kvm_session_release_of, kvm_browser_close_of, + KvmJournalRead, KvmJournalText, KvmJournalAbsent, KvmJournalUnread, kvm_journal_read_at, kvm_journal_events_of, kvm_last_result, + megarac_kvm_observer_start, megarac_kvm_observer_finish, megarac_kvm_still_allowance, + kvm_observer_record_text, +} + +// THE REAL KVM OBSERVER AGAINST AN ADMITTED MEGARAC CONTROLLER, WITH NO BOOT (operator-approved brief, +// 2026-09-30; re-rooted onto the admitted carrier by managed-host cut 5). The observer +// (gunbc.machine_intake_megarac_kvm_still) is the boot's; this probe drives the SAME functions the boot +// drives -- megarac_kvm_observer_start, the handoff admission that waits for establishment, and +// megarac_kvm_observer_finish, which asks one terminal still, requests the stop, releases the web +// session and the browser and hashes the still -- and nothing else. +// +// MEGARAC-SCOPED BY REFUSAL: the probe takes gunbc.megarac_managed_host AdmittedMegaRacHost, which a host +// of another family or an unobserved one never mints, and megarac_kvm_observer_start refuses a carrier +// admitted for any operation but the KVM canvas still. The controller, its account and the host whose +// unit hold the probe takes all come off that carrier; the artifact directory is the caller's. +// +// NOTHING IS WRITTEN TO THE CONTROLLER'S MEDIA, POWER, BOOT DEVICE OR SEL. The web session it opens is +// its own and is DELETEd on every reachable exit; the only other effect is the KVM viewer seat, which +// is why the probe holds the unit (gunbc.managed_host_unit_hold kvm_observer_acquire_unit_hold): the +// boot takes that seat under the same hold and refuses its handoff when the seat is busy, so an unheld +// probe could make a concurrent boot refuse. +// +// THE ARTIFACTS are the observer's own directory -- journal, journal.pre-teardown, observer.log and +// every still-N.jpg -- under the caller's directory prefix, plus the probe's receipt, which is +// kvm_observer_record_text of the record, written on every path that started the observer. + +// WHERE A PROBE'S ARTIFACTS LAND: an observer directory is the caller's prefix followed by the attempt, +// and a probe's attempt carries megarac_kvm_observer_probe_prefix, so its directories, and the receipt +// beside them, are one glob apart from every boot's under the same prefix. +type KvmObserverProbePaths { + dir_prefix: String +} + +data megarac_kvm_observer_probe_prefix: String = "probe" + +fn megarac_kvm_observer_probe_attempt(run_id: NonEmptyStr) -> String { + join([megarac_kvm_observer_probe_prefix, "-", run_id as String], "") +} + +fn megarac_kvm_observer_dir(paths: KvmObserverProbePaths, attempt: String) -> String { + concat(paths.dir_prefix, attempt) +} + +fn megarac_kvm_observer_probe_receipt_path(paths: KvmObserverProbePaths) -> String { + concat(megarac_kvm_observer_dir(paths: paths, attempt: megarac_kvm_observer_probe_prefix), "-receipt.txt") +} + +// Everything the probe keeps: every probe's observer directory and its receipt. +fn megarac_kvm_observer_probe_artifact_glob(paths: KvmObserverProbePaths) -> String { + concat(megarac_kvm_observer_dir(paths: paths, attempt: megarac_kvm_observer_probe_prefix), "-*") +} + +// WHAT THE PROBE'S OWN BOUNDED WAITS COST AT MOST is the observer's own allowance +// (megarac_kvm_still_allowance: the establishment wait, the terminal-still wait, the stop wait and +// the owned release), counted once. +fn megarac_kvm_observer_probe_bound() -> Second { + megarac_kvm_still_allowance() +} + +// THE VERDICT IS A CONJUNCTION OF OBSERVED FACTS, EACH READ FROM THE RECORD: established for this +// host and attempt (the handoff admission's own test), the one terminal still answered and hashed, and +// the teardown seen through -- stopped line written, session released, browser closed, process gone. +type KvmProbeVerdict + = KvmProbeHeld + | KvmProbeFell { reason: NonEmptyStr } + +// THE TEARDOWN IS JUDGED BY ITS WORDS, NOT BY SUBSTRINGS: the session release must read exactly +// `released` and the browser close exactly `closed` (kvm_session_release_of / kvm_browser_close_of, the +// journal's own RS rows), the stopped line must have been seen, and the owned process released or +// already gone. A "release-failed detail=..." whose detail mentions "released" is a failed release. +// Absent means seen through; Present names the first fact that was not. +fn kvm_probe_teardown_gap(rec: KvmObserverRecord) -> NonEmptyStr? { + match rec.teardown { + KvmTeardownRecorded { terminal: _, snapshot: _, requested_at: _, stopped_seen: st, session_release: sr, browser_close: bc, process: p } => + if !st { Present { value: "the observer's stopped line was not seen" as NonEmptyStr } } + else if !(match kvm_session_release_of(result: sr) { KvmSessionReleased => true _ => false }) { Present { value: concat("the web session was not released: ", sr) as NonEmptyStr } } + else if !(match kvm_browser_close_of(result: bc) { KvmBrowserClosed => true _ => false }) { Present { value: concat("the browser was not closed: ", bc) as NonEmptyStr } } + else if !(match p { OwnedReleased { pid: _ } => true OwnedAlreadyGone { pid: _ } => true _ => false }) { Present { value: "the observer process was not seen released" as NonEmptyStr } } + else { none } + KvmTeardownNotNeeded { reason: r } => Present { value: concat("no teardown was recorded: ", r as String) as NonEmptyStr } + } +} + +fn kvm_probe_still_answered(rec: KvmObserverRecord) -> Bool { + match rec.teardown { + KvmTeardownRecorded { terminal: t, snapshot: _, requested_at: _, stopped_seen: _, session_release: _, browser_close: _, process: _ } => + match t { KvmTerminalStillAnswered { line: _ } => true _ => false } + _ => false + } +} + +fn kvm_probe_hashed_stills(rec: KvmObserverRecord) -> Int { + count(filter(rec.stills, s => match s { KvmStillHashed { at: _, seq: _, reason: _, path: _, digest: _ } => true _ => false })) +} + +fn kvm_probe_verdict(admission: String, established: Bool, rec: KvmObserverRecord) -> KvmProbeVerdict { + if !established { KvmProbeFell { reason: concat("the observer was not established: ", admission) as NonEmptyStr } } + else if !kvm_probe_still_answered(rec: rec) { KvmProbeFell { reason: "the terminal still was not answered" } } + else if kvm_probe_hashed_stills(rec: rec) < 1 { KvmProbeFell { reason: "no still was retained with its digest" } } + else { + match kvm_probe_teardown_gap(rec: rec) { + Present { value: g } => KvmProbeFell { reason: g } + Absent => KvmProbeHeld + } + } +} + +// THE RECEIPT NAMES THE ADMITTED CONTROLLER, never a unit constant, then the verdict and the record. +fn kvm_probe_receipt_text(subject: NonEmptyStr, verdict: KvmProbeVerdict, rec: KvmObserverRecord) -> String { + let verdict_line = match verdict { KvmProbeHeld => "verdict=held" KvmProbeFell { reason: r } => concat("verdict=FELL: ", r as String) } + join([concat("subject=", subject as String), verdict_line, kvm_observer_record_text(rec: rec)], "\n") +} + +fn kvm_probe_conclude(subject: NonEmptyStr, paths: KvmObserverProbePaths, rec: KvmObserverRecord, verdict: KvmProbeVerdict) -> ProcessExit { + let kept = Filesystem.Write(path: megarac_kvm_observer_probe_receipt_path(paths: paths), content: concat(kvm_probe_receipt_text(subject: subject, verdict: verdict, rec: rec), "\n")) + let outcome = match verdict { + KvmProbeHeld => ExitSuccess + KvmProbeFell { reason: r } => exit_failure(reason: concat("megarac kvm observer: ", r as String)) + } + if kept.success { outcome } else { + match outcome { + ExitSuccess => exit_failure(reason: concat("megarac kvm observer: the probe held and its receipt could not be written: ", kept.error)) + other => other + } + } +} + +// WHAT THE HELD PROBE LEAVES: its exit, and the record its seat decision is read from. +type KvmProbeHeldRun { + outcome: ProcessExit + record: KvmObserverRecord +} + +fn megarac_kvm_observer_observe_held(host: AdmittedMegaRacHost, password_file: NonEmptyStr, run_id: NonEmptyStr, paths: KvmObserverProbePaths) -> KvmProbeHeldRun + admit_callers: [ + decl_ref(module_path: "gunbc.machine_intake_megarac_kvm_observer_observe", decl_name: "megarac_kvm_observer_observe"), + ] +{ + let attempt = megarac_kvm_observer_probe_attempt(run_id: run_id) + let start = megarac_kvm_observer_start( + host: host, + dir: megarac_kvm_observer_dir(paths: paths, attempt: attempt), + password_file: password_file as String, + toolchain: runner_browser_toolchain_here_wet(), + attempt: attempt, + ) + let admission = kvm_handoff_admission(start: start) + let established = match admission { KvmHandoffAdmitted { session: _ } => true KvmHandoffRefused { cause: _ } => false } + let admission_text = match admission { KvmHandoffAdmitted { session: s } => concat("established, web session ", s) KvmHandoffRefused { cause: c } => c } + let rec = megarac_kvm_observer_finish(start: start, attempt: attempt, terminal_order: 1) + let subject = host.binding.observation.endpoint.host + KvmProbeHeldRun { outcome: kvm_probe_conclude(subject: subject, paths: paths, rec: rec, verdict: kvm_probe_verdict(admission: admission_text, established: established, rec: rec)), record: rec } +} + +// THE PROBE OVER AN ADMITTED HOST: the unit hold is taken on the host the carrier names, the observer +// is driven, and the hold is freed only over a seat observed let go. Sealed to the one entry that binds a +// managed host's row to it; it returns a ProcessExit, never a carrier. +fn megarac_kvm_observer_observe(host: AdmittedMegaRacHost, password_file: NonEmptyStr, run_id: NonEmptyStr, paths: KvmObserverProbePaths) -> ProcessExit + admit_callers: [ + decl_ref(module_path: "gunbc.machine_intake_mtcollins1_kvm_observer_observe", decl_name: "mtcollins1_kvm_observer_observe_wet"), + ] +{ + match kvm_observer_acquire_unit_hold(host: admitted_megarac_host_identity(host: host), run_id: run_id) { + UnitHoldRefused { reason: r } => exit_failure(reason: concat("megarac kvm observer: ", r as String)) + UnitHeld { proof: proof } => { + let run = megarac_kvm_observer_observe_held(host: host, password_file: password_file, run_id: run_id, paths: paths) + match kvm_probe_seat_hold_gap(rec: run.record) { + Absent => unit_hold_release(proof: proof, outcome: run.outcome) + Present { value: gap } => exit_failure(reason: kvm_probe_hold_kept_text(gap: gap, outcome: run.outcome)) + } + } + } +} + +// THE UNIT HOLD IS FREED ONLY OVER A SEAT OBSERVED LET GO (side-chat review, blocker 4a). The hold is +// what keeps a boot from starting while the probe may still own the BMC's one KVM viewer seat, so it is +// released only when every part of that ownership is observed ended, by its typed word: +// - the web session reads `released`, or `no-session` (no login was ever accepted, so none exists); +// - the browser reads `closed`, or `no-browser` (none was launched); +// - the observer process is observed exited (released, already gone, or its pid now names another +// process), or was never recorded because the observer never launched. +// `login-unobserved` (a login may have been accepted), any failed, http- or unrecognised word, an +// unresolved process or an unreadable journal KEEPS the hold and says why; the BMC's own session +// timeout frees the seat, and the hold is then cleared by the operator or, once it lands, by +// observed-dead recovery. Absent is "let go"; Present names the first part not observed ended. +fn kvm_probe_seat_parts_gap(session: KvmSessionRelease, browser: KvmBrowserClose, process: OwnedRelease, launched: Bool) -> NonEmptyStr? { + let session_ended = match session { KvmSessionReleased => true KvmSessionNoSession => true KvmSessionReleaseHttp { status: _ } => false KvmSessionReleaseFailed { detail: _ } => false KvmSessionLoginUnobserved => false KvmSessionReleaseUnrecognized { raw: _ } => false } + let browser_ended = match browser { KvmBrowserClosed => true KvmBrowserNotLaunched => true KvmBrowserCloseFailed { detail: _ } => false KvmBrowserCloseUnrecognized { raw: _ } => false } + let process_ended = match process { + OwnedReleased { pid: _ } => true + OwnedAlreadyGone { pid: _ } => true + OwnedReleaseFailed { pid: _, detail: _ } => false + OwnedReleaseUnresolved { pid: _, standing: _ } => false + OwnedReleaseNotOwned { standing: s } => match s { OwnedForeign { pid: _, observed_start: _ } => true OwnedNotRecorded { detail: _ } => !launched OwnedRunning { pid: _, start_time: _ } => false OwnedGone { pid: _ } => false OwnedUnreadable { pid: _, detail: _ } => false OwnedUnparseable { pid: _, detail: _ } => false } + } + if !session_ended { Present { value: "the web session was not observed released" as NonEmptyStr } } + else if !browser_ended { Present { value: "the browser was not observed closed" as NonEmptyStr } } + else if !process_ended { Present { value: concat("the observer process was not observed exited: ", owned_release_text(r: process)) as NonEmptyStr } } + else { none } +} + +fn kvm_probe_seat_hold_gap(rec: KvmObserverRecord) -> NonEmptyStr? { + match rec.teardown { + KvmTeardownNotNeeded { reason: _ } => none + KvmTeardownRecorded { terminal: _, snapshot: _, requested_at: _, stopped_seen: _, session_release: sr, browser_close: bc, process: p } => + kvm_probe_seat_parts_gap(session: kvm_session_release_of(result: sr), browser: kvm_browser_close_of(result: bc), process: p, launched: true) + } +} + +fn kvm_probe_hold_kept_text(gap: NonEmptyStr, outcome: ProcessExit) -> String { + join(["megarac kvm observer: unit hold KEPT, because ", gap as String, "; the probe's own outcome was: ", match outcome { ExitSuccess => "success" ExitFailure { code: _, reason: r } => r }], "") +} + +// THE SAME RULE FROM THE ARTIFACTS ALONE, for the always() step: the journal's last session-release and +// browser-close words and the owned release it just performed. A journal that cannot be read keeps the +// hold; no journal and no pid record means the observer never launched. +fn kvm_probe_seat_gap_from_journal(journal: KvmJournalRead, process: OwnedRelease) -> NonEmptyStr? { + match journal { + KvmJournalUnread { cause: c } => Present { value: concat("the observer journal could not be read: ", c) as NonEmptyStr } + KvmJournalAbsent => kvm_probe_seat_parts_gap(session: KvmSessionNoSession, browser: KvmBrowserNotLaunched, process: process, launched: false) + KvmJournalText { text: _ } => { + let events = kvm_journal_events_of(read: journal) + kvm_probe_seat_parts_gap(session: kvm_session_release_of(result: kvm_last_result(events: events, session: true)), browser: kvm_browser_close_of(result: kvm_last_result(events: events, session: false)), process: process, launched: true) + } + } +} + +// THE always() RELEASE (a separate step after the probe's, run whatever the probe step did: success, +// failure, cancel or its own timeout). First the observer process this run started, by its recorded +// (pid, start time) -- so the viewer seat and the web session it holds are let go -- then the unit hold +// on `host`, only if THIS run holds it (gunbc.managed_host_unit_hold kvm_observer_release_unit_hold). A +// probe that finished cleanly left nothing, and both halves report that as a no-op. It fails loudly only +// when something this run owns stays held. It contacts no controller, so it takes the host's identity +// rather than an admitted carrier: a release must still run when the host no longer admits the probe. +fn megarac_kvm_observer_release(host: HostIdentity, run_id: NonEmptyStr, paths: KvmObserverProbePaths) -> ProcessExit + admit_callers: [ + decl_ref(module_path: "gunbc.machine_intake_mtcollins1_kvm_observer_observe", decl_name: "mtcollins1_kvm_observer_release_wet"), + ] +{ + let dir = megarac_kvm_observer_dir(paths: paths, attempt: megarac_kvm_observer_probe_attempt(run_id: run_id)) + let process = owned_process_release(pid_path: concat(dir, "/observer.pid")) + match kvm_probe_seat_gap_from_journal(journal: kvm_journal_read_at(dir: dir), process: process) { + Present { value: gap } => return exit_failure(reason: join(["megarac kvm observer release: process ", owned_release_text(r: process), "; unit hold KEPT, because ", gap as String], "")) + Absent => {} + } + let hold = kvm_observer_release_unit_hold(host: host, run_id: run_id) + let line = join(["megarac kvm observer release: process ", owned_release_text(r: process), "; unit hold ", probe_hold_release_outcome_text(o: hold)], "") + match hold { + ProbeHoldUnreleased { cause: _ } => exit_failure(reason: line) + _ => ExitSuccess + } +} diff --git a/dag/gunbc/machine_intake/mtcollins1_fan_observe.dag b/dag/gunbc/machine_intake/mtcollins1_fan_observe.dag index 1f67849e0bd..23556e462b2 100644 --- a/dag/gunbc/machine_intake/mtcollins1_fan_observe.dag +++ b/dag/gunbc/machine_intake/mtcollins1_fan_observe.dag @@ -2,49 +2,23 @@ module gunbc.machine_intake_mtcollins1_fan_observe import extdeps.shell import std.algebra { trim } -import std.decl_ref { DeclarationRef, WholeDeclaration } -import std.types { Int, List, NonEmptyStr, String } -import std.measure { ObservationRoundCount, Second, observation_round_count, percent, second, second_count } -import std.process { ExitSuccess, ProcessExit, exit_failure } -import extdeps.filesystem.filesystem_io { Filesystem } -import product.host_identity { HostIdentity } -import product.fan_policy_standing { - FanDutyCeilingVerdict, - ExcursionBound, - FanDutyExcursionOverCeiling, - FanDutyNotReported, - NoPostTransitionWindow, - OutlivedPostTransitionWindow, - WithinPostTransitionWindow, - FanDutySteadyCurveOverCeiling, - FanDutyWithinCeiling, - FanPolicyAuthority, - FanPolicySample, - FanPolicyStanding, - fan_policy_standing_label, - fan_policy_standing_of_sample, -} -import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint } -import gunbc.managed_host { managed_host_account_driving, managed_host_binding_for } +import std.types { NonEmptyStr, String } +import std.measure { ObservationRoundCount, Second, observation_round_count, second } +import std.process { ProcessExit, exit_failure } +import std.optional { Present, Absent } +import gunbc.managed_host { managed_host_binding_for } import gunbc.fleet_host_identity { operator_host_mtcollins1 } -import gunbc.machine_intake_bmc_fan_observation { - FanObservationRefused, - FanObservationRound, - FanObservationSeries, - FanObservationTaken, - elapsed_at_round, - first_posted_round_index, - observe_fan_policy, - render_series, - samples_of_round, +import gunbc.machine_intake_fan_observe { + FanObserveAdmitted, FanObserveRefused, FanObservePolicy, + fan_observe_admission, fan_observe_refusal_text, fan_observe_with, } -import gunbc.fleet_acoustic_intent { residence_duty_verdict } -// THE POLICY LAYER FOR ONE HOST'S FAN OBSERVATION, separate from the producer for the same reason -// the media-attach and boot-handoff wrappers are separate from theirs. Three things are decided here -// and nowhere else: how many samples this unit takes and how far apart, WHICH FAMILY AUTHORITY -// governs its controller, and which site ceiling bounds it. A producer carrying any of those would -// need a second copy for Mt. Jade. +// MT. COLLINS UNIT 1'S BINDING OF THE FAN OBSERVATION (managed-host cut 5). The read, the family-selected +// policy authority and the receipt's rendering are gunbc.machine_intake_fan_observe over the host's +// managed binding; what stays here is what the fleet-converge workflow names (ruling (a): the workflow +// surface moves in cut 7) -- the entry function, the credential variable its prelude materializes and +// the receipt path its upload reads -- and this unit's own series policy: how many samples, how far +// apart, and the window its standings are judged in. data mtcollins1_fan_credential_path_env: NonEmptyStr = "GUNBC_MTCOLLINS1_BMC_CREDENTIAL_FILE" as NonEmptyStr @@ -61,19 +35,6 @@ data mtcollins1_fan_observation_receipt_path: String = "target/mtcollins1-fan-ob data mtcollins1_fan_observation_rounds: ObservationRoundCount = observation_round_count(count: 6) data mtcollins1_fan_observation_interval: Second = second(count: 10) -// THIS UNIT'S CONTROLLER IS MEGARAC, SO ITS FAN POLICY AUTHORITY IS THE MEGARAC ONE - which is the -// module that says, in its own type, that nothing has been read. Pointing at it is not a claim that -// its policy is known; it is a claim about WHERE the answer will live when somebody reads the -// vendor documentation, and it is what makes the standing's provenance re-derivable from the -// artifact rather than from this sentence. -data mtcollins1_fan_policy_authority: FanPolicyAuthority = FanPolicyAuthority { - declaration: DeclarationRef { - module_path: "extdeps.bmc.megarac_fan_policy", - decl_name: "megarac_fan_policy_standing", - field: WholeDeclaration - } -} - // A MINUTE IS NOT A TRANSITION WINDOW ANYBODY MEASURED, and this row says so rather than implying // otherwise. No family publishes how long its loop takes to close, and this run is sixty seconds // long, so a window at sixty seconds means "this series cannot distinguish a slow close from a held @@ -112,95 +73,11 @@ fn classify_fan_credential_path(raw: String) -> MtCollins1FanCredentialPath { } } -// THE COMPARISON DUTY IS ONE HUNDRED PERCENT, AND THAT IS THIS RUN'S QUESTION RATHER THAN A FACT -// ABOUT MEGARAC. extdeps.bmc.megarac_fan_policy publishes no failsafe duty and this module may not -// invent one for it. What passing 100 asks is narrower and defensible - "is the drive at maximum" - -// which is exactly the state the operator described on Mt. Jade. If the vendor documentation later -// names a lower failsafe target, this call site moves and every standing re-derives. It is spelled -// as a literal rather than a row because percent() admits a literal only (std.measure -// percent_from_computed_int_frontier), so a row here would be a name that could not be passed. -// -// NINETY-FIVE PERCENT IS THIS LANE'S STATED TOLERANCE, NOT A FACT ABOUT ANY FAN, and what it decides -// is narrower than it looks. It selects between NearCatalogMaximum and BelowCatalogMaximum on a -// SPEED observation; it cannot reach a failsafe verdict, because no tolerance converts RPM into a -// commanded duty (operator ruling 2026-09-18: speed at a given PWM depends on supply voltage, part -// tolerance, static pressure and chassis impedance). -// -// It is inert on Mt. Collins today for a further reason worth stating: no fan part is modeled for -// this unit, so catalog_maximum is Absent and the relation comes back CatalogMaximumUnknown before -// the tolerance is consulted at all. It is supplied rather than defaulted so that the day a fan part -// lands, the number it is compared against is a value somebody chose and can revise. -// -// ONE STANDING PER SAMPLE, EACH WITH THE SITE DUTY VERDICT BESIDE IT. Both lines are rendered for -// every sample rather than summarized, because a summary would be a second authority for the same -// reading and the whole point of the artifact is that the verdict can be re-derived from it. -fn render_sample_standing(sample: FanPolicySample, elapsed: Second?) -> String { - let standing = fan_policy_standing_of_sample( - host: mtcollins1_endpoint.host as HostIdentity, - authority: mtcollins1_fan_policy_authority, - sample: sample, - failsafe_duty: percent(100), - catalog_maximum: Absent, - at_maximum_tolerance: percent(95), - elapsed_since_post: elapsed, - transition_window: mtcollins1_fan_transition_window, - ) - join([ - " standing fan=", sample.endpoint.name as String, - " at=", sample.observed_at as String, - " verdict=", fan_policy_standing_label(standing: standing) as String, - " residence_duty=", render_duty_verdict(verdict: residence_duty_verdict(standing: standing)), - ], "") -} - -// THE TWO OVER-ARMS RENDER DISTINCTLY, because their remedies differ: a curve over the ceiling is -// something to converge, and an excursion over it is something to bound in time or to end by -// powering the machine off. -fn render_duty_verdict(verdict: FanDutyCeilingVerdict) -> String { - match verdict { - FanDutyWithinCeiling { duty: _, ceiling: _ } => "WithinCeiling" - FanDutySteadyCurveOverCeiling { duty: _, ceiling: _ } => "SteadyCurveOverCeiling" - FanDutyExcursionOverCeiling { duty: _, ceiling: _, bound: b } => join(["ExcursionOverCeiling(", render_excursion_bound(bound: b), ")"], "") - FanDutyNotReported { cause: _ } => "NotReported" - } -} - -// EACH ROUND'S SAMPLES ARE STAMPED WITH THAT ROUND'S OWN ELAPSED, never with the series'. The first -// posted index is computed once because it is a property of the series; the elapsed is computed per -// round from it, because that is a property of the round. -// THE BOUND RENDERS AS WHAT IT IS, and the pre-POST case says so in words rather than as a zero. -// This receipt is what the operator reads about his own room, and "window=0s" said an excursion had -// a zero-second budget where the fact is that nothing bounds it but him powering the machine off. -fn render_excursion_bound(bound: ExcursionBound) -> String { - match bound { - NoPostTransitionWindow => "no post-transition window; bounded only by power-off" - WithinPostTransitionWindow { elapsed: e, window: w } => - join(["within window, elapsed=", to_string(second_count(s: e)), "s of ", to_string(second_count(s: w)), "s"], "") - OutlivedPostTransitionWindow { elapsed: e, window: w } => - join(["OUTLIVED window, elapsed=", to_string(second_count(s: e)), "s past ", to_string(second_count(s: w)), "s"], "") - } -} - -fn render_observation(rounds: List, interval: Second) -> String { - let first_posted = first_posted_round_index(rounds: rounds) - join([ - render_series(rounds: rounds), - join( - map(rounds, r => render_round_standings(round: r, first_posted: first_posted, interval: interval)), - "\n", - ), - ], "\n") -} - -fn render_round_standings(round: FanObservationRound, first_posted: Int?, interval: Second) -> String { - let elapsed = elapsed_at_round(round: round, first_posted_index: first_posted, interval: interval) - join( - map( - samples_of_round(host: mtcollins1_endpoint.host as HostIdentity, round: round), - s => render_sample_standing(sample: s, elapsed: elapsed), - ), - "\n", - ) +data mtcollins1_fan_observe_policy: FanObservePolicy = FanObservePolicy { + rounds: mtcollins1_fan_observation_rounds, + interval: mtcollins1_fan_observation_interval, + transition_window: mtcollins1_fan_transition_window, + receipt_path: mtcollins1_fan_observation_receipt_path, } // REFUSALS BEFORE THE FIRST REQUEST, EACH NAMING WHAT IS MISSING. A run that reaches the controller @@ -208,58 +85,16 @@ fn render_round_standings(round: FanObservationRound, first_posted: Int?, interv // machine fact is how "the fans could not be observed" becomes "the fans are not reporting". fn mtcollins1_fan_observe_wet() -> ProcessExit { - match managed_host_account_driving(lookup: managed_host_binding_for(host: operator_host_mtcollins1 as NonEmptyStr), endpoint: mtcollins1_endpoint) { - Absent => - return exit_failure(reason: "mtcollins1 fan observation: the managed host does not bind (not BmcSecured, or its observed controller is not the secured one), so this run holds no account to read sensors with; contacting NO host") - Present { value: username } => + match fan_observe_admission(lookup: managed_host_binding_for(host: operator_host_mtcollins1 as NonEmptyStr)) { + FanObserveRefused { cause: c } => exit_failure(reason: concat("mtcollins1 fan observation: ", fan_observe_refusal_text(cause: c) as String)) + FanObserveAdmitted { host: admitted } => match mtcollins1_fan_credential_path() { FanCredentialPathUnset => - return exit_failure( - reason: join([ - "mtcollins1 fan observation: no ", - mtcollins1_fan_credential_path_env as String, - " in the environment, so this run materialized no controller credential; contacting NO host", - ], "") - ) + exit_failure(reason: join(["mtcollins1 fan observation: no ", mtcollins1_fan_credential_path_env as String, " in the environment, so this run materialized no controller credential; contacting NO host"], "")) FanCredentialPathEmpty => - return exit_failure( - reason: join([ - "mtcollins1 fan observation: ", - mtcollins1_fan_credential_path_env as String, - " is set but empty, which is not a path - the prelude ran and materialized nothing; contacting NO host", - ], "") - ) + exit_failure(reason: join(["mtcollins1 fan observation: ", mtcollins1_fan_credential_path_env as String, " is set but empty, which is not a path - the prelude ran and materialized nothing; contacting NO host"], "")) FanCredentialPathReady { path: password_file } => - return mtcollins1_write_observation( - series: observe_fan_policy( - bmc_host: mtcollins1_endpoint.host, - username: username, - password_file: password_file, - rounds: mtcollins1_fan_observation_rounds, - interval: mtcollins1_fan_observation_interval, - ) - ) - } - } -} - -// THE RECEIPT IS WRITTEN ONLY FOR A SERIES THAT WAS TAKEN. Writing a refused observation's text -// would leave an artifact that LOOKS like a measurement attached to a run that failed, and the -// upload step would attach it with no way for a later reader to tell. -fn mtcollins1_write_observation(series: FanObservationSeries) -> ProcessExit { - match series { - FanObservationRefused { cause: cause } => - exit_failure(reason: concat("mtcollins1 fan observation: ", cause as String)) - FanObservationTaken { rounds: rounds, interval: interval } => { - let written = Filesystem.Write( - path: mtcollins1_fan_observation_receipt_path, - content: render_observation(rounds: rounds, interval: interval), - ) - if written.success { - ExitSuccess - } else { - exit_failure(reason: concat("mtcollins1 fan observation: the series was taken and its receipt could not be written: ", written.error)) + fan_observe_with(host: admitted, password_file: password_file, policy: mtcollins1_fan_observe_policy) } - } } } diff --git a/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag b/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag index ea2c52c9eeb..85625d1d651 100644 --- a/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag +++ b/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag @@ -1,175 +1,57 @@ module gunbc.machine_intake_mtcollins1_kvm_observer_observe -import std.types { Bool, Int, List, NonEmptyStr } +import std.types { NonEmptyStr, String } import std.algebra { trim } -import std.measure { Second, second, second_count } -import std.process { ExitSuccess, ExitFailure, ProcessExit, exit_failure } -import std.optional { Present, Absent } -import extdeps.filesystem.filesystem_io { Filesystem } +import std.measure { Second } +import std.process { ExitSuccess, ProcessExit, exit_failure } import extdeps.github.actions_environment { github_run_id_variable_name } import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent } -import gunbc.runner_browser_toolchain { runner_browser_toolchain_here_wet } -import gunbc.owned_process { OwnedRelease, OwnedReleased, OwnedAlreadyGone, OwnedReleaseNotOwned, OwnedNotRecorded, OwnedForeign, owned_process_release, owned_release_text } -import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint } -import gunbc.megarac_managed_host { AdmittedMegaRacHost, MegaRacHostAdmitted, MegaRacHostRefused, megarac_host_admission_for, megarac_host_refusal_text } +import gunbc.megarac_managed_host { MegaRacHostAdmitted, MegaRacHostRefused, megarac_host_admission_for, megarac_host_refusal_text } import gunbc.megarac_operation_standing { MegaRacKvmCanvasStill } import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { mtcollins1_kvm_observer_dir } import gunbc.machine_intake_mtcollins1_fan_observe { FanCredentialPathReady, FanCredentialPathUnset, FanCredentialPathEmpty, mtcollins1_fan_credential_path, mtcollins1_fan_credential_path_env } import gunbc.machine_intake_mtcollins1_boot_admission { mtcollins1_boot_host_admission } import gunbc.fleet_host_identity { operator_host_mtcollins1 } -import gunbc.managed_host_unit_hold { - UnitHeld, UnitHoldRefused, kvm_observer_acquire_unit_hold, unit_hold_release, - ProbeHoldReleased, ProbeHoldAlreadyFree, ProbeHoldNotOurs, ProbeHoldStale, ProbeHoldUnreleased, - kvm_observer_release_unit_hold, probe_hold_release_outcome_text, +import gunbc.machine_intake_megarac_kvm_observer_observe { + KvmObserverProbePaths, megarac_kvm_observer_observe, megarac_kvm_observer_release, + megarac_kvm_observer_probe_artifact_glob, megarac_kvm_observer_probe_bound, } -import gunbc.machine_intake_megarac_kvm_still { - KvmObserverRecord, KvmObserverStart, KvmObserverStarted, KvmObserverNotStarted, - KvmHandoffAdmitted, KvmHandoffRefused, kvm_handoff_admission, - KvmStillHashed, KvmTeardownRecorded, KvmTeardownNotNeeded, KvmTerminalStillAnswered, - KvmSessionRelease, KvmSessionReleased, KvmSessionNoSession, KvmBrowserClose, KvmBrowserClosed, KvmBrowserNotLaunched, kvm_session_release_of, kvm_browser_close_of, - KvmJournalRead, KvmJournalText, KvmJournalAbsent, KvmJournalUnread, kvm_journal_read_at, kvm_journal_events_of, kvm_last_result, - megarac_kvm_observer_start, megarac_kvm_observer_finish, megarac_kvm_still_allowance, - kvm_observer_record_text, -} - -// THE REAL KVM OBSERVER AGAINST THE REAL BMC, WITH NO BOOT (operator-approved brief, 2026-09-30). -// The observer (gunbc.machine_intake_megarac_kvm_still) is the boot's; this mode drives the SAME -// functions the boot drives -- megarac_kvm_observer_start, the handoff admission that waits for -// establishment, and megarac_kvm_observer_finish, which asks one terminal still, requests the stop, -// releases the web session and the browser and hashes the still -- and nothing else. So it exercises, -// on the hardware, exactly the login and the MegaRAC SPA navigation after DOMContentLoaded that ended -// run 36721915217, and the journal records where the page went (`navigated` lines, written and read -// only through kvm_journal_line / KvmJournalWord). -// -// NOTHING IS WRITTEN TO THE CONTROLLER'S MEDIA, POWER, BOOT DEVICE OR SEL. The web session it opens is -// its own and is DELETEd on every reachable exit; the only other effect is the KVM viewer seat, which -// is why the run holds the unit (gunbc.managed_host_unit_hold -// kvm_observer_acquire_unit_hold): the boot takes that seat under the same hold and refuses -// its handoff when the seat is busy, so an unheld probe could make a concurrent boot refuse. -// -// THE ARTIFACTS are the observer's own directory -- journal, journal.pre-teardown, observer.log and -// every still-N.jpg -- under mtcollins1_kvm_observer_dir(attempt), plus this mode's receipt, which is -// kvm_observer_record_text of the record, written on every path that started the observer. - -// The attempt names the observer's directory (mtcollins1_kvm_observer_dir); a probe's carries this -// prefix, so its directories, and the receipt beside them, are one glob apart from every boot's. -data mtcollins1_kvm_observer_probe_prefix: String = "probe" -fn mtcollins1_kvm_observer_probe_attempt(run_id: NonEmptyStr) -> String { - join([mtcollins1_kvm_observer_probe_prefix, "-", run_id as String], "") -} +// MT. COLLINS UNIT 1'S BINDING OF THE KVM OBSERVER PROBE (managed-host cut 5). The probe itself is +// gunbc.machine_intake_megarac_kvm_observer_observe over an admitted MegaRAC host; what stays here is +// only what the fleet-converge workflow names (ruling (a): the workflow surface moves in cut 7) -- the +// two entry functions, the artifact glob and the step bound -- and the mtcollins1 row it binds: the host +// label, the boot's observer-directory authority this unit's probes share a prefix with, the executor +// host the unit hold store lives on, the workflow run id that owns the hold, and the credential file the +// workflow prelude materializes. -data mtcollins1_kvm_observer_observe_receipt_path: String = concat(mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_prefix), "-receipt.txt") +// The probe's directories share the boot's observer prefix (gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle +// mtcollins1_kvm_observer_dir), derived from that authority rather than re-spelled. +data mtcollins1_kvm_observer_probe_paths: KvmObserverProbePaths = KvmObserverProbePaths { dir_prefix: mtcollins1_kvm_observer_dir(attempt: "") } -// Everything the probe keeps: every probe's observer directory and its receipt. -data mtcollins1_kvm_observer_probe_artifact_glob: String = concat(mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_prefix), "-*") +data mtcollins1_kvm_observer_probe_artifact_glob: String = megarac_kvm_observer_probe_artifact_glob(paths: mtcollins1_kvm_observer_probe_paths) -// WHAT THE PROBE'S OWN BOUNDED WAITS COST AT MOST is the observer's own allowance -// (megarac_kvm_still_allowance: the establishment wait, the terminal-still wait, the stop wait and -// the owned release), counted once. The step budget adds gunbc run's load and the credential prelude to -// this on the workflow side, which owns those authorities. fn mtcollins1_kvm_observer_probe_bound() -> Second { - megarac_kvm_still_allowance() -} - -// THE VERDICT IS A CONJUNCTION OF OBSERVED FACTS, EACH READ FROM THE RECORD: established for this -// host and attempt (the handoff admission's own test), the one terminal still answered and hashed, and -// the teardown seen through -- stopped line written, session released, browser closed, process gone. -type KvmProbeVerdict - = KvmProbeHeld - | KvmProbeFell { reason: NonEmptyStr } - -// THE TEARDOWN IS JUDGED BY ITS WORDS, NOT BY SUBSTRINGS: the session release must read exactly -// `released` and the browser close exactly `closed` (kvm_session_release_of / kvm_browser_close_of, the -// journal's own RS rows), the stopped line must have been seen, and the owned process released or -// already gone. A "release-failed detail=..." whose detail mentions "released" is a failed release. -// Absent means seen through; Present names the first fact that was not. -fn kvm_probe_teardown_gap(rec: KvmObserverRecord) -> NonEmptyStr? { - match rec.teardown { - KvmTeardownRecorded { terminal: _, snapshot: _, requested_at: _, stopped_seen: st, session_release: sr, browser_close: bc, process: p } => - if !st { Present { value: "the observer's stopped line was not seen" as NonEmptyStr } } - else if !(match kvm_session_release_of(result: sr) { KvmSessionReleased => true _ => false }) { Present { value: concat("the web session was not released: ", sr) as NonEmptyStr } } - else if !(match kvm_browser_close_of(result: bc) { KvmBrowserClosed => true _ => false }) { Present { value: concat("the browser was not closed: ", bc) as NonEmptyStr } } - else if !(match p { OwnedReleased { pid: _ } => true OwnedAlreadyGone { pid: _ } => true _ => false }) { Present { value: "the observer process was not seen released" as NonEmptyStr } } - else { none } - KvmTeardownNotNeeded { reason: r } => Present { value: concat("no teardown was recorded: ", r as String) as NonEmptyStr } - } -} - -fn kvm_probe_still_answered(rec: KvmObserverRecord) -> Bool { - match rec.teardown { - KvmTeardownRecorded { terminal: t, snapshot: _, requested_at: _, stopped_seen: _, session_release: _, browser_close: _, process: _ } => - match t { KvmTerminalStillAnswered { line: _ } => true _ => false } - _ => false - } + megarac_kvm_observer_probe_bound() } -fn kvm_probe_hashed_stills(rec: KvmObserverRecord) -> Int { - count(filter(rec.stills, s => match s { KvmStillHashed { at: _, seq: _, reason: _, path: _, digest: _ } => true _ => false })) -} - -fn kvm_probe_verdict(admission: String, established: Bool, rec: KvmObserverRecord) -> KvmProbeVerdict { - if !established { KvmProbeFell { reason: concat("the observer was not established: ", admission) as NonEmptyStr } } - else if !kvm_probe_still_answered(rec: rec) { KvmProbeFell { reason: "the terminal still was not answered" } } - else if kvm_probe_hashed_stills(rec: rec) < 1 { KvmProbeFell { reason: "no still was retained with its digest" } } - else { - match kvm_probe_teardown_gap(rec: rec) { - Present { value: g } => KvmProbeFell { reason: g } - Absent => KvmProbeHeld - } - } -} - -fn kvm_probe_conclude(rec: KvmObserverRecord, verdict: KvmProbeVerdict) -> ProcessExit { - let verdict_line = match verdict { KvmProbeHeld => "verdict=held" KvmProbeFell { reason: r } => concat("verdict=FELL: ", r as String) } - let body = join([concat("subject=", mtcollins1_endpoint.host as String), verdict_line, kvm_observer_record_text(rec: rec)], "\n") - let kept = Filesystem.Write(path: mtcollins1_kvm_observer_observe_receipt_path, content: concat(body, "\n")) - let outcome = match verdict { - KvmProbeHeld => ExitSuccess - KvmProbeFell { reason: r } => exit_failure(reason: concat("mtcollins1 kvm observer: ", r as String)) - } - if kept.success { outcome } else { - match outcome { - ExitSuccess => exit_failure(reason: concat("mtcollins1 kvm observer: the probe held and its receipt could not be written: ", kept.error)) - other => other - } +// THE RUN ID THAT OWNS THE UNIT HOLD, read from the workflow run; empty when unset. +fn mtcollins1_kvm_observer_run_id() -> String { + match actions_variable_read(name: github_run_id_variable_name) { + ActionsVariablePresent { value: v } => trim(s: v) + ActionsVariableAbsent { variable: _, detail: _ } => "" } } -// WHAT THE HELD PROBE LEAVES: its exit, and the record its seat decision is read from. -type KvmProbeHeldRun { - outcome: ProcessExit - record: KvmObserverRecord -} - -fn mtcollins1_kvm_observer_observe_held(host: AdmittedMegaRacHost, password_file: NonEmptyStr, run_id: NonEmptyStr) -> KvmProbeHeldRun { - let attempt = mtcollins1_kvm_observer_probe_attempt(run_id: run_id) - let start = megarac_kvm_observer_start( - host: host, - dir: mtcollins1_kvm_observer_dir(attempt: attempt), - password_file: password_file as String, - toolchain: runner_browser_toolchain_here_wet(), - attempt: attempt, - ) - let admission = kvm_handoff_admission(start: start) - let established = match admission { KvmHandoffAdmitted { session: _ } => true KvmHandoffRefused { cause: _ } => false } - let admission_text = match admission { KvmHandoffAdmitted { session: s } => concat("established, web session ", s) KvmHandoffRefused { cause: c } => c } - let rec = megarac_kvm_observer_finish(start: start, attempt: attempt, terminal_order: 1) - KvmProbeHeldRun { outcome: kvm_probe_conclude(rec: rec, verdict: kvm_probe_verdict(admission: admission_text, established: established, rec: rec)), record: rec } -} - // REFUSALS BEFORE THE FIRST REQUEST, EACH NAMING WHAT IS MISSING: off srv1 (the hold store's host), -// no secured account, no credential file, no run id, or the unit already held. +// no run id, the host not admitted for the KVM canvas still, or no credential file. fn mtcollins1_kvm_observer_observe_wet() -> ProcessExit { match mtcollins1_boot_host_admission() { ExitSuccess => {} other => return other } - let run_id = match actions_variable_read(name: github_run_id_variable_name) { - ActionsVariablePresent { value: v } => trim(s: v) - ActionsVariableAbsent { variable: _, detail: _ } => "" - } + let run_id = mtcollins1_kvm_observer_run_id() if run_id == "" { return exit_failure(reason: concat(concat("mtcollins1 kvm observer: ", github_run_id_variable_name as String), " is unset or empty, so the unit hold has no owner to name; contacting NO host")) } @@ -180,98 +62,16 @@ fn mtcollins1_kvm_observer_observe_wet() -> ProcessExit FanCredentialPathUnset => exit_failure(reason: join(["mtcollins1 kvm observer: no ", mtcollins1_fan_credential_path_env as String, " in the environment; contacting NO host"], "")) FanCredentialPathEmpty => exit_failure(reason: join(["mtcollins1 kvm observer: ", mtcollins1_fan_credential_path_env as String, " is set but empty; contacting NO host"], "")) FanCredentialPathReady { path: p } => - match kvm_observer_acquire_unit_hold(host: operator_host_mtcollins1, run_id: run_id as NonEmptyStr) { - UnitHoldRefused { reason: r } => exit_failure(reason: concat("mtcollins1 kvm observer: ", r as String)) - UnitHeld { proof: proof } => { - let run = mtcollins1_kvm_observer_observe_held(host: admitted, password_file: p, run_id: run_id as NonEmptyStr) - match kvm_probe_seat_hold_gap(rec: run.record) { - Absent => unit_hold_release(proof: proof, outcome: run.outcome) - Present { value: gap } => exit_failure(reason: kvm_probe_hold_kept_text(gap: gap, outcome: run.outcome)) - } - } - } + megarac_kvm_observer_observe(host: admitted, password_file: p, run_id: run_id as NonEmptyStr, paths: mtcollins1_kvm_observer_probe_paths) } } } -// THE UNIT HOLD IS FREED ONLY OVER A SEAT OBSERVED LET GO (side-chat review, blocker 4a). The hold is -// what keeps a boot from starting while the probe may still own the BMC's one KVM viewer seat, so it is -// released only when every part of that ownership is observed ended, by its typed word: -// - the web session reads `released`, or `no-session` (no login was ever accepted, so none exists); -// - the browser reads `closed`, or `no-browser` (none was launched); -// - the observer process is observed exited (released, already gone, or its pid now names another -// process), or was never recorded because the observer never launched. -// `login-unobserved` (a login may have been accepted), any failed, http- or unrecognised word, an -// unresolved process or an unreadable journal KEEPS the hold and says why; the BMC's own session -// timeout frees the seat, and the hold is then cleared by the operator or, once it lands, by -// observed-dead recovery. Absent is "let go"; Present names the first part not observed ended. -fn kvm_probe_seat_parts_gap(session: KvmSessionRelease, browser: KvmBrowserClose, process: OwnedRelease, launched: Bool) -> NonEmptyStr? { - let session_ended = match session { KvmSessionReleased => true KvmSessionNoSession => true KvmSessionReleaseHttp { status: _ } => false KvmSessionReleaseFailed { detail: _ } => false KvmSessionLoginUnobserved => false KvmSessionReleaseUnrecognized { raw: _ } => false } - let browser_ended = match browser { KvmBrowserClosed => true KvmBrowserNotLaunched => true KvmBrowserCloseFailed { detail: _ } => false KvmBrowserCloseUnrecognized { raw: _ } => false } - let process_ended = match process { - OwnedReleased { pid: _ } => true - OwnedAlreadyGone { pid: _ } => true - OwnedReleaseFailed { pid: _, detail: _ } => false - OwnedReleaseUnresolved { pid: _, standing: _ } => false - OwnedReleaseNotOwned { standing: s } => match s { OwnedForeign { pid: _, observed_start: _ } => true OwnedNotRecorded { detail: _ } => !launched OwnedRunning { pid: _, start_time: _ } => false OwnedGone { pid: _ } => false OwnedUnreadable { pid: _, detail: _ } => false OwnedUnparseable { pid: _, detail: _ } => false } - } - if !session_ended { Present { value: "the web session was not observed released" as NonEmptyStr } } - else if !browser_ended { Present { value: "the browser was not observed closed" as NonEmptyStr } } - else if !process_ended { Present { value: concat("the observer process was not observed exited: ", owned_release_text(r: process)) as NonEmptyStr } } - else { none } -} - -fn kvm_probe_seat_hold_gap(rec: KvmObserverRecord) -> NonEmptyStr? { - match rec.teardown { - KvmTeardownNotNeeded { reason: _ } => none - KvmTeardownRecorded { terminal: _, snapshot: _, requested_at: _, stopped_seen: _, session_release: sr, browser_close: bc, process: p } => - kvm_probe_seat_parts_gap(session: kvm_session_release_of(result: sr), browser: kvm_browser_close_of(result: bc), process: p, launched: true) - } -} - -fn kvm_probe_hold_kept_text(gap: NonEmptyStr, outcome: ProcessExit) -> String { - join(["mtcollins1 kvm observer: unit hold KEPT, because ", gap as String, "; the probe's own outcome was: ", match outcome { ExitSuccess => "success" ExitFailure { code: _, reason: r } => r }], "") -} - -// THE SAME RULE FROM THE ARTIFACTS ALONE, for the always() step: the journal's last session-release and -// browser-close words and the owned release it just performed. A journal that cannot be read keeps the -// hold; no journal and no pid record means the observer never launched. -fn kvm_probe_seat_gap_from_journal(journal: KvmJournalRead, process: OwnedRelease) -> NonEmptyStr? { - match journal { - KvmJournalUnread { cause: c } => Present { value: concat("the observer journal could not be read: ", c) as NonEmptyStr } - KvmJournalAbsent => kvm_probe_seat_parts_gap(session: KvmSessionNoSession, browser: KvmBrowserNotLaunched, process: process, launched: false) - KvmJournalText { text: _ } => { - let events = kvm_journal_events_of(read: journal) - kvm_probe_seat_parts_gap(session: kvm_session_release_of(result: kvm_last_result(events: events, session: true)), browser: kvm_browser_close_of(result: kvm_last_result(events: events, session: false)), process: process, launched: true) - } - } -} - -// THE always() RELEASE (a separate step after the probe's, run whatever the probe step did: success, -// failure, cancel or its own timeout). First the observer process this run started, by its recorded -// (pid, start time) -- so the viewer seat and the web session it holds are let go -- then the unit hold, -// only if THIS run holds it (gunbc.managed_host_unit_hold -// kvm_observer_release_unit_hold). A probe that finished cleanly left nothing, and both -// halves report that as a no-op. It fails loudly only when something this run owns stays held. fn mtcollins1_kvm_observer_release_wet() -> ProcessExit { - let run_id = match actions_variable_read(name: github_run_id_variable_name) { - ActionsVariablePresent { value: v } => trim(s: v) - ActionsVariableAbsent { variable: _, detail: _ } => "" - } + let run_id = mtcollins1_kvm_observer_run_id() if run_id == "" { return exit_failure(reason: concat(concat("mtcollins1 kvm observer release: ", github_run_id_variable_name as String), " is unset or empty, so this run cannot name the hold it may release")) } - let dir = mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_attempt(run_id: run_id as NonEmptyStr)) - let process = owned_process_release(pid_path: concat(dir, "/observer.pid")) - match kvm_probe_seat_gap_from_journal(journal: kvm_journal_read_at(dir: dir), process: process) { - Present { value: gap } => return exit_failure(reason: join(["mtcollins1 kvm observer release: process ", owned_release_text(r: process), "; unit hold KEPT, because ", gap as String], "")) - Absent => {} - } - let hold = kvm_observer_release_unit_hold(host: operator_host_mtcollins1, run_id: run_id as NonEmptyStr) - let line = join(["mtcollins1 kvm observer release: process ", owned_release_text(r: process), "; unit hold ", probe_hold_release_outcome_text(o: hold)], "") - match hold { - ProbeHoldUnreleased { cause: _ } => exit_failure(reason: line) - _ => ExitSuccess - } + megarac_kvm_observer_release(host: operator_host_mtcollins1, run_id: run_id as NonEmptyStr, paths: mtcollins1_kvm_observer_probe_paths) } diff --git a/dag/gunbc/non_fold_residue.dag b/dag/gunbc/non_fold_residue.dag index e9e36c9f22c..500eb5a7db8 100644 --- a/dag/gunbc/non_fold_residue.dag +++ b/dag/gunbc/non_fold_residue.dag @@ -678,10 +678,10 @@ data non_fold_residue_frontier: List = [ FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_boot_run.dag::mtcollins1_boot_sol_is_held" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_boot_run.dag::mtcollins1_sol_await_observer" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_boot_run.dag::sol_await_establishment_until" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, - FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag::kvm_probe_hashed_stills" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, - FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag::kvm_probe_still_answered" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, - FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag::kvm_probe_teardown_gap" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, - FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag::mtcollins1_kvm_observer_release_wet" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, + FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag::kvm_probe_hashed_stills" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, + FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag::kvm_probe_still_answered" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, + FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag::kvm_probe_teardown_gap" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, + FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/megarac_kvm_observer_observe.dag::megarac_kvm_observer_release" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_sol_notice.dag::sol_notice_annotation" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_sol_notice.dag::sol_notice_deliver" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, FrontierRow { subject: PathSubject { path: "dag/gunbc/machine_intake/mtcollins1_sol_notice.dag::sol_notice_observe" }, reason: nfr_reason_typed_census_ca5ed1724b, dissolution: nfr_dissolve_owning_fold }, diff --git a/dag/test/claim/machine_intake/bmc_stack_observation_forged_probe_witness_test.dag b/dag/test/claim/machine_intake/bmc_stack_observation_forged_probe_witness_test.dag new file mode 100644 index 00000000000..cfd0f76ed49 --- /dev/null +++ b/dag/test/claim/machine_intake/bmc_stack_observation_forged_probe_witness_test.dag @@ -0,0 +1,70 @@ +module test.claim.machine_intake.bmc_stack_observation_forged_probe_witness_test + +import v2.std.algebra { filter } +import std.types { String, Bool, Int, List } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import gunbc.compile_diagnostic_census { + CompileDiagnosticCensus, CompileDiagnosticCensusRow, CensusObserved, CensusNotRunnable, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// THE MANAGED-HOST CUT 5 SEALS, ENROLLED AS EXECUTED REDS. The probe source below authors a FanObserveHost +// outside its mint, and calls each admitted body of the fan read and the MegaRAC KVM probe -- and the +// effects beneath them -- from outside their admitted callers, each with a carrier or raw values in hand. +// Served-UI observation is not in this public tree (f956099fcbe moved AMI-bundle-derived content to +// gunbc-private), so that seal is not re-authored here. The witness hands that source to the compiler +// through the diagnostic-census harness and requires the refusal CLASS at each SUBJECT by name. +data forged_fan_source: String = "module probe_bmc_stack_observation_forged_fan\n\nimport std.types { NonEmptyStr }\nimport std.process { ProcessExit }\nimport extdeps.bmc.access_profile { BmcAccessObservation }\nimport product.host_identity { HostIdentity }\nimport product.fan_policy_standing { FanPolicyAuthority }\nimport gunbc.machine_intake_fan_observe { FanObserveHost, FanObservePolicy, fan_observe_with }\n\nfn forged_fan_host(o: BmcAccessObservation, h: HostIdentity, a: FanPolicyAuthority) -> FanObserveHost {\n FanObserveHost { host: h, observation: o, account: \"admin\" as NonEmptyStr, authority: a }\n}\n\nfn fan_read_from_outside(h: FanObserveHost, p: FanObservePolicy) -> ProcessExit {\n fan_observe_with(host: h, password_file: \"/tmp/pw\" as NonEmptyStr, policy: p)\n}\n" +data forged_kvm_source: String = "module probe_bmc_stack_observation_forged_kvm\n\nimport std.types { NonEmptyStr }\nimport std.process { ProcessExit }\nimport product.host_identity { HostIdentity }\nimport gunbc.megarac_managed_host { AdmittedMegaRacHost }\nimport gunbc.machine_intake_megarac_kvm_observer_observe { KvmObserverProbePaths, KvmProbeHeldRun, megarac_kvm_observer_observe, megarac_kvm_observer_observe_held, megarac_kvm_observer_release }\n\nfn kvm_probe_from_outside(h: AdmittedMegaRacHost) -> ProcessExit {\n megarac_kvm_observer_observe(host: h, password_file: \"/tmp/pw\" as NonEmptyStr, run_id: \"1\" as NonEmptyStr, paths: KvmObserverProbePaths { dir_prefix: \"x-\" })\n}\n\nfn kvm_held_from_outside(h: AdmittedMegaRacHost) -> KvmProbeHeldRun {\n megarac_kvm_observer_observe_held(host: h, password_file: \"/tmp/pw\" as NonEmptyStr, run_id: \"1\" as NonEmptyStr, paths: KvmObserverProbePaths { dir_prefix: \"x-\" })\n}\n\nfn kvm_release_from_outside(h: HostIdentity) -> ProcessExit {\n megarac_kvm_observer_release(host: h, run_id: \"1\" as NonEmptyStr, paths: KvmObserverProbePaths { dir_prefix: \"x-\" })\n}\n" + +fn probe(source: String) -> CompileDiagnosticCensus { + compile_dag_diagnostic_census(source) +} + +fn blocking_count_for_class_and_subject(c: CompileDiagnosticCensus, wanted: String, subject: String) -> Int { + match c { + CensusNotRunnable { cause: _ } => -1 + CensusObserved { rows: rows } => + rows + |> filter(r => (r.diagnostic_class as String) == wanted && r.subject_name == subject && r.blocking) + |> fold(init: 0, f: (acc, r) => acc + r.count) + } +} + +fn blocking_count_for_class(c: CompileDiagnosticCensus, wanted: String) -> Int { + match c { + CensusNotRunnable { cause: _ } => -1 + CensusObserved { rows: rows } => + rows |> filter(r => (r.diagnostic_class as String) == wanted && r.blocking) |> fold(init: 0, f: (acc, r) => acc + r.count) + } +} + +data harness_control_source: String = "module probe_bmc_stack_observation_harness_control\nimport std.types { NonEmptyStr }\nimport gunbc.managed_host { managed_host_binding_for }\nimport gunbc.machine_intake_fan_observe { FanObserveAdmission, fan_observe_admission }\nfn admission(host: NonEmptyStr) -> FanObserveAdmission { fan_observe_admission(lookup: managed_host_binding_for(host: host)) }\n" + +// THE CONTROL IS CLASS-SCOPED: a clean source reaching the fan admission through its real entry yields +// NO row of either class the reds count. +test fn the_harness_runs_and_a_clean_source_emits_neither_wall_class() -> Bool { + let c = compile_dag_diagnostic_census(harness_control_source) + blocking_count_for_class(c: c, wanted: "SoleConstructorViolation") == 0 + && blocking_count_for_class(c: c, wanted: "ConstructorCallAdmissionRefused") == 0 +} + +// A FAN-READ CARRIER CANNOT BE AUTHORED OUTSIDE ITS MINT, so no controller, account or policy authority +// reaches the fan read except through a managed binding. +test fn a_forged_fan_observe_host_is_refused_at_its_literal() -> Bool { + blocking_count_for_class_and_subject(c: probe(source: forged_fan_source), wanted: "SoleConstructorViolation", subject: "FanObserveHost") >= 1 +} + +// EACH ADMITTED BODY, AND EACH EFFECT BENEATH IT, REFUSES AN OUTSIDE CALLER -- even one holding a carrier. +test fn the_fan_read_refuses_an_outside_caller() -> Bool { + blocking_count_for_class_and_subject(c: probe(source: forged_fan_source), wanted: "ConstructorCallAdmissionRefused", subject: "fan_observe_with") >= 1 +} + +// THE KVM PROBE, ITS HELD RUN AND ITS RELEASE refuse an outside caller. +test fn the_kvm_probe_and_its_release_refuse_an_outside_caller() -> Bool { + let c = probe(source: forged_kvm_source) + blocking_count_for_class_and_subject(c: c, wanted: "ConstructorCallAdmissionRefused", subject: "megarac_kvm_observer_observe") >= 1 + && blocking_count_for_class_and_subject(c: c, wanted: "ConstructorCallAdmissionRefused", subject: "megarac_kvm_observer_observe_held") >= 1 + && blocking_count_for_class_and_subject(c: c, wanted: "ConstructorCallAdmissionRefused", subject: "megarac_kvm_observer_release") >= 1 +} diff --git a/dag/test/claim/machine_intake/fan_observe_witness_test.dag b/dag/test/claim/machine_intake/fan_observe_witness_test.dag new file mode 100644 index 00000000000..52b6885da42 --- /dev/null +++ b/dag/test/claim/machine_intake/fan_observe_witness_test.dag @@ -0,0 +1,105 @@ +module test.claim.machine_intake.fan_observe_witness_test + +import std.types { Bool, NonEmptyStr, String } +import std.optional { Present, Absent } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import extdeps.bmc.endpoint { OpenBmc } +import extdeps.bmc.capability { bmc_firmware_release_identity_of_wire } +import extdeps.bmc.access_profile { BmcAccessObservation, BmcAccessObserved, BmcAccessUnobserved, BmcObservedIdentity } +import gunbc.fleet_host_identity { operator_host_mtcollins1 } +import gunbc.managed_host { ManagedHostFound, managed_host_binding, managed_host_binding_for, managed_host_fixture, managed_hosts } +import gunbc.machine_intake_fan_observe { + FanObserveAdmission, FanObserveAdmitted, FanObserveRefused, FanHostFamilyUnobserved, FanHostNotManaged, + fan_observe_admission, fan_observe_authority, fan_observe_refusal_text, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +fn authority_names(a: FanObserveAdmission, module_path: String, decl_name: String) -> Bool { + match a { + FanObserveAdmitted { host: h } => + (fan_observe_authority(host: h).declaration.module_path as String) == module_path + && (fan_observe_authority(host: h).declaration.decl_name as String) == decl_name + FanObserveRefused { cause: _ } => false + } +} + +// THE INHABITANCE CLAIM, AND IT ASSERTS THE ROUTE: the one managed row, mtcollins1, reaches the fan read +// through its real binding (gunbc.managed_host managed_host_binding_for, the lookup the wet entry +// makes), and its observed family -- AMI MegaRAC -- selects the MegaRAC policy authority. Its observed +// family changing, its binding refusing, or the dispatch defaulting turns this red. +test fn the_megarac_row_is_read_against_the_megarac_policy() -> Bool { + let a = fan_observe_admission(lookup: managed_host_binding_for(host: operator_host_mtcollins1 as NonEmptyStr)) + authority_names(a: a, module_path: "extdeps.bmc.megarac_fan_policy", decl_name: "megarac_fan_policy_standing") + && match a { + FanObserveAdmitted { host: h } => + (h.host as String) == (operator_host_mtcollins1 as String) + && managed_hosts() |> all(r => match r.access { + BmcAccessObserved { observation: o } => (o.endpoint.host as String) == (h.observation.endpoint.host as String) + BmcAccessUnobserved { endpoint: _ } => false + }) + && (h.account as String) == "gunbc" + FanObserveRefused { cause: _ } => false + } +} + +fn observation_as_openbmc(o: BmcAccessObservation) -> BmcAccessObservation { + BmcAccessObservation { + endpoint: o.endpoint, + identity: BmcObservedIdentity { board: o.identity.board, chassis_part_number: o.identity.chassis_part_number, firmware: bmc_firmware_release_identity_of_wire(family: OpenBmc, wire: o.identity.firmware.raw_version) }, + discovered_surfaces: o.discovered_surfaces, + boot_control_routes: o.boot_control_routes, + redfish_virtual_media_probe: o.redfish_virtual_media_probe, + successful_read_surfaces: o.successful_read_surfaces, + successful_write_surfaces: o.successful_write_surfaces, + raw_response: o.raw_response, + observed_at: o.observed_at, + } +} + +// A SUPPLIED CONTROL, NOT A BINDING: no managed host is OpenBMC-bound. The real row with ONLY its observed +// family changed selects the phosphor zone contract, so the authority follows the observed family and is +// not a constant of the module. +test fn an_openbmc_bound_row_is_read_against_the_phosphor_policy() -> Bool { + match managed_hosts().first() { + Absent => false + Present { value: h } => + match h.access { + BmcAccessUnobserved { endpoint: _ } => false + BmcAccessObserved { observation: o } => { + let openbmc = managed_host_fixture(host: h.host, access: BmcAccessObserved { observation: observation_as_openbmc(o: o) }, capability_row: h.capability_row, secure: h.secure, unit_hold: h.unit_hold, smpro: h.smpro) + authority_names(a: fan_observe_admission(lookup: ManagedHostFound { standing: managed_host_binding(host: openbmc) }), module_path: "extdeps.bmc.phosphor_fan_control", decl_name: "phosphor_fan_zone_policy_contract") + } + } + } +} + +// RED: THE SAME ROW WITH ITS CONTROLLER UNOBSERVED (Mt. Jade's standing). There is no family to select a +// policy from, so the read refuses with a cause naming the controller whose observation is missing -- it +// is never read against either policy by default. +test fn a_row_whose_family_is_unobserved_refuses_naming_the_missing_observation() -> Bool { + match managed_hosts().first() { + Absent => false + Present { value: h } => + match h.access { + BmcAccessUnobserved { endpoint: _ } => false + BmcAccessObserved { observation: o } => { + let unobserved = managed_host_fixture(host: h.host, access: BmcAccessUnobserved { endpoint: o.endpoint }, capability_row: h.capability_row, secure: h.secure, unit_hold: h.unit_hold, smpro: h.smpro) + match fan_observe_admission(lookup: ManagedHostFound { standing: managed_host_binding(host: unobserved) }) { + FanObserveRefused { cause: FanHostFamilyUnobserved { endpoint: e } } => + (e.host as String) == (o.endpoint.host as String) + && string_contains(s: fan_observe_refusal_text(cause: FanHostFamilyUnobserved { endpoint: e }) as String, pattern: "was never observed") + _ => false + } + } + } + } +} + +// RED: A HOST THE FLEET DOES NOT MANAGE. mtjade1 has no managed row, so its fan read refuses by name. +test fn a_host_that_is_not_managed_refuses() -> Bool { + match fan_observe_admission(lookup: managed_host_binding_for(host: "mtjade1" as NonEmptyStr)) { + FanObserveRefused { cause: FanHostNotManaged { host: n } } => (n as String) == "mtjade1" + _ => false + } +} diff --git a/dag/test/claim/machine_intake/mtcollins1_kvm_observer_observe_witness_test.dag b/dag/test/claim/machine_intake/megarac_kvm_observer_observe_witness_test.dag similarity index 80% rename from dag/test/claim/machine_intake/mtcollins1_kvm_observer_observe_witness_test.dag rename to dag/test/claim/machine_intake/megarac_kvm_observer_observe_witness_test.dag index 4f01a2bb035..f2519c70e22 100644 --- a/dag/test/claim/machine_intake/mtcollins1_kvm_observer_observe_witness_test.dag +++ b/dag/test/claim/machine_intake/megarac_kvm_observer_observe_witness_test.dag @@ -1,6 +1,7 @@ -module test.claim.machine_intake.mtcollins1_kvm_observer_observe_witness_test +module test.claim.machine_intake.megarac_kvm_observer_observe_witness_test import std.types { Bool, List, NonEmptyStr, String } +import std.optional { Present, Absent } import std.measure { second, millisecond } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.crypto.hash { sha256_digest } @@ -14,17 +15,20 @@ import gunbc.machine_intake_megarac_kvm_still { KvmSessionReleased, KvmSessionReleaseHttp, KvmSessionReleaseFailed, KvmSessionLoginUnobserved, KvmSessionNoSession, KvmSessionReleaseUnrecognized, KvmBrowserClosed, KvmBrowserNotLaunched, KvmBrowserCloseFailed, KvmBrowserCloseUnrecognized, kvm_session_release_of, kvm_browser_close_of, } -import gunbc.machine_intake_mtcollins1_kvm_observer_observe { - KvmProbeVerdict, KvmProbeHeld, KvmProbeFell, kvm_probe_verdict, - mtcollins1_kvm_observer_probe_attempt, mtcollins1_kvm_observer_observe_receipt_path, mtcollins1_kvm_observer_probe_artifact_glob, +import gunbc.machine_intake_mtcollins1_kvm_observer_observe { mtcollins1_kvm_observer_probe_paths, mtcollins1_kvm_observer_probe_artifact_glob } +import gunbc.machine_intake_megarac_kvm_observer_observe { + KvmProbeVerdict, KvmProbeHeld, KvmProbeFell, kvm_probe_verdict, kvm_probe_receipt_text, + KvmObserverProbePaths, megarac_kvm_observer_probe_attempt, megarac_kvm_observer_probe_receipt_path, megarac_kvm_observer_probe_artifact_glob, megarac_kvm_observer_dir, kvm_probe_seat_parts_gap, kvm_probe_seat_hold_gap, kvm_probe_seat_gap_from_journal, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly // THE VERDICT IS READ FROM THE RECORD megarac_kvm_observer_finish RETURNS, so its inputs are SUPPLIED -// records (DESIGN §3: a witness discriminates at one interface). No public KVM client drives the -// observer (rung drop handoff_observer_is_sol_not_kvm_viewer), so its start is a typed refusal. +// records (DESIGN §3: a witness discriminates at one interface). The real producer runs twice elsewhere: +// against the loopback controller in test.claim.machine_intake.mtcollins1_kvm_observer_protocol_wet_witness, +// which holds the same finish, stop and release to what the controller saw, and against the real BMC +// in the fleet-converge mode this module's subject belongs to. fn at() -> KvmInstant { KvmInstant { since_epoch: millisecond(count: 1790000000000), iso: "2026-09-30T20:00:00.000Z" } @@ -129,13 +133,42 @@ test fn every_teardown_word_reads_into_its_own_arm() -> Bool { // THE ARTIFACT GLOB CARRIES WHAT THE PROBE WRITES: its observer directory and its receipt, both derived // from the observer's own directory authority, and not a boot's directory. test fn the_upload_glob_covers_the_probe_directory_and_receipt_and_no_boot_directory() -> Bool { - let dir = mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_attempt(run_id: "36800000000" as NonEmptyStr)) + let p = mtcollins1_kvm_observer_probe_paths + let dir = megarac_kvm_observer_dir(paths: p, attempt: megarac_kvm_observer_probe_attempt(run_id: "36800000000" as NonEmptyStr)) let prefix = substring(s: mtcollins1_kvm_observer_probe_artifact_glob, start: 0, end: string_length(mtcollins1_kvm_observer_probe_artifact_glob) - 1) starts_with(s: dir, prefix: prefix) - && starts_with(s: mtcollins1_kvm_observer_observe_receipt_path, prefix: prefix) + && starts_with(s: megarac_kvm_observer_probe_receipt_path(paths: p), prefix: prefix) && !starts_with(s: mtcollins1_kvm_observer_dir(attempt: "36800000000"), prefix: prefix) } +// THE BINDING'S PREFIX IS THE BOOT'S OBSERVER-DIRECTORY AUTHORITY, so a probe directory is a boot +// directory's sibling under one prefix and the workflow's glob is unchanged by the re-root. +test fn the_mtcollins1_probe_paths_are_derived_from_the_boots_observer_directory() -> Bool { + megarac_kvm_observer_dir(paths: mtcollins1_kvm_observer_probe_paths, attempt: "probe-1") == mtcollins1_kvm_observer_dir(attempt: "probe-1") + && mtcollins1_kvm_observer_probe_artifact_glob == "artifacts/mtcollins1-boot-kvm-probe-*" +} + +// THE DIRECTORY AND THE RECEIPT ARE THE CALLER'S PATHS, NEVER A UNIT CONSTANT: two prefixes differing in +// nothing else land the same attempt in two places. +test fn the_probe_directory_and_receipt_follow_the_supplied_prefix() -> Bool { + let a = KvmObserverProbePaths { dir_prefix: "artifacts/host-a-kvm-" } + let b = KvmObserverProbePaths { dir_prefix: "artifacts/host-b-kvm-" } + megarac_kvm_observer_dir(paths: a, attempt: "probe-7") == "artifacts/host-a-kvm-probe-7" + && megarac_kvm_observer_dir(paths: b, attempt: "probe-7") == "artifacts/host-b-kvm-probe-7" + && megarac_kvm_observer_probe_receipt_path(paths: a) == "artifacts/host-a-kvm-probe-receipt.txt" + && megarac_kvm_observer_probe_artifact_glob(paths: b) == "artifacts/host-b-kvm-probe-*" +} + +// THE RECEIPT NAMES THE ADMITTED CONTROLLER: the same verdict and record under two subjects differ in the +// subject line and nowhere else. +test fn the_receipt_subject_is_the_admitted_controller() -> Bool { + let a = kvm_probe_receipt_text(subject: "192.0.2.10" as NonEmptyStr, verdict: KvmProbeHeld, rec: held_record()) + let b = kvm_probe_receipt_text(subject: "192.0.2.20" as NonEmptyStr, verdict: KvmProbeHeld, rec: held_record()) + starts_with(s: a, prefix: "subject=192.0.2.10\nverdict=held\n") + && starts_with(s: b, prefix: "subject=192.0.2.20\nverdict=held\n") + && substring(s: a, start: string_length("subject=192.0.2.10"), end: string_length(a)) == substring(s: b, start: string_length("subject=192.0.2.20"), end: string_length(b)) +} + // ── THE UNIT HOLD IS FREED ONLY OVER A SEAT OBSERVED LET GO (side-chat blocker 4a) ───────────────────── fn keeps(g: NonEmptyStr?, pattern: String) -> Bool { match g { Present { value: why } => string_contains(s: why as String, pattern: pattern) Absent => false }