diff --git a/dag/gunbc/host/host_budget_source.dag b/dag/gunbc/host/host_budget_source.dag index 2cb18c62a96..7898e4e7b69 100644 --- a/dag/gunbc/host/host_budget_source.dag +++ b/dag/gunbc/host/host_budget_source.dag @@ -333,8 +333,8 @@ fn kernel_budget_sourcing_static_source(g: KernelBudgetSourcing) -> HostBudgetSo // may bound a refusal; it may not stand in for a reading. // THE CGROUP-v1 ARM JOINS THIS SAME DECLARED MIRROR rather than opening a second one. The seed's -// memory_governor CgroupV1HierarchicalMemoryLimit arm, CgroupV1MemoryLimitValue, -// cgroup_v1_hierarchical_limit_from_stat and cgroup_v1_unlimited_bytes hand-realize +// v1_rt HostBudgetCgroupV1 arm, host_budget_cgroup_v1_from_stat and +// host_budget_cgroup_v1_unlimited_bytes hand-realize // BudgetSourceCgroupV1HierarchicalMemoryLimit here and extdeps.linux.cgroup_v1_memory's parse, under // the bootstrap constraint stated in the note below: the budget bounds the resolve, so it is read // before any .dag value exists. The mirror keeps the model's three states (limited / unlimited / diff --git a/dag/gunbc/rung_drop/fixture_closure_union_unmodeled_stderr_capture.dag b/dag/gunbc/rung_drop/fixture_closure_union_unmodeled_stderr_capture.dag index e9df5634113..0b70ff52377 100644 --- a/dag/gunbc/rung_drop/fixture_closure_union_unmodeled_stderr_capture.dag +++ b/dag/gunbc/rung_drop/fixture_closure_union_unmodeled_stderr_capture.dag @@ -1,29 +1,14 @@ module gunbc.rung_drop.fixture_closure_union_unmodeled_stderr_capture import std.types { NonEmptyStr } -import gunbc.rung_drop { RungDrop, Standing, TypedDeclaration, ReplacementStaged } +import gunbc.rung_drop { RungDrop, Retired, TypedDeclaration, ReplacementStaged } import gunbc.guarantee_rung { Mitigatable, MechanicallyPreventable } -import gunbc.stderr_capture_gap_exclusion { exclusion, StderrCaptureGapExclusion } - -// THE FIXTURE-CLOSURE UNION RENDERS EVERY RECORDED MEMBER THROUGH THE RUST EMITTER. After -// #13437 the walker closes by reference, so a fixture that reaches extdeps.gunbc compiles -// gunbc.WitnessBin.Run into the union. That operation declares stderr_truncated / -// stderr_total_bytes / stderr_retained_bytes, which v1.compiler.emit shell_channel_realized_by_target -// refuses for rust: the emitted Command body implements no WitnessStderrCapturePolicy. -// emit_artifact then returns no files for the WHOLE union, so those three diagnostics abort -// every other member's render and refuse unrelated floors (specimen: #13420). -// -// THE EARLIEST UNJUSTIFIED BOUNDARY IS THE UNION, NOT THE WALL. The wall is the honest -// capability gap (05_emit names the next-rung trigger: the emitted realization implementing -// the declared policy). std.shell_stream_capture still homes the policy on the interpreter -// drain until that trigger fires. Binding a fabricating rust handler (truncated=false) is the -// fail-open 05_emit already refuses. So the union must not render the unmodeled operation: -// a typed exclusion keyed on that diagnostic fact, never a silent skip, and a real emit error -// in a member the union still renders still refuses. - -fn capture_gap_exclusion_population() -> StderrCaptureGapExclusion { - exclusion -} + +// RETIRED — TRIGGER FIRED. gunbc#13472 realized rust shell WitnessStderrCapturePolicy +// (v1.compiler.emit shell_channel_realized_by_target is true for the three capture- +// accounting channels). The union no longer strips WitnessBin.Run. Discriminating +// controls stay enrolled: fixture_closure_union_controls gunbc-reach emit, plus +// emit_host realization tests (DESIGN §4b(4)). data fixture_closure_union_unmodeled_stderr_capture: RungDrop = RungDrop { identity: "fixture_closure_union_unmodeled_stderr_capture" as NonEmptyStr, @@ -32,7 +17,9 @@ data fixture_closure_union_unmodeled_stderr_capture: RungDrop = RungDrop { declared: "2026-10-06", - standing: Standing, + standing: Retired { + trigger_fired: "2026-10-08 -- gunbc#13472. THE CAPABILITY, EXECUTED: v1.compiler.emit shell_channel_realized_by_target returns true for ShellChanStderrTruncated, ShellChanStderrTotalBytes and ShellChanStderrRetainedBytes on rust because emit_shell_call implements the declared WitnessStderrCapturePolicy. WitnessBin.Run emits without TransportEmissionNotModeled; the union strip of that service is deleted; this exclusion population is empty. DISCRIMINATING CONTROL, ENROLLED: v1_compiler.cli_run fixture_closure_union_controls admits a fixture whose closure reaches extdeps.gunbc, and a real emit error beside that reach still refuses as FixtureClosureUnionEmitRefused at module=efr_member. Answering the channels as untruncated whole-stream lengths did not discharge this. Deleting the channels from the interface did not discharge this." as NonEmptyStr + }, declaration: TypedDeclaration { previous: MechanicallyPreventable, @@ -41,9 +28,7 @@ data fixture_closure_union_unmodeled_stderr_capture: RungDrop = RungDrop { reason: ReplacementStaged { replacement: "the rust emitted realization implementing the declared WitnessStderrCapturePolicy, the same next-rung trigger v1.compiler.emit shell_channel_realized_by_target already names" }, - population: [ - "exactly gunbc.stderr_capture_gap_exclusion.exclusion (consumed here as capture_gap_exclusion_population): gunbc.WitnessBin.Run in extdeps.gunbc. Output channels stderr_truncated, stderr_total_bytes, stderr_retained_bytes, each a TransportEmissionNotModeled ShellChannelNotRealizedByTarget for target rust. A later shell operation with the same capture-gap fact is NOT a member: the union still renders it and still refuses. The union compiles and evaluates that record; deleting either this module or the carrier refuses the seed compile", - ], + population: [], restoration_trigger: "THE CAPABILITY: the rust emitted shell realization implements the operation's declared WitnessStderrCapturePolicy, sufficient for v1.compiler.emit shell_channel_realized_by_target to return true for ShellChanStderrTruncated, ShellChanStderrTotalBytes and ShellChanStderrRetainedBytes. WHAT THAT MUST BE SUFFICIENT FOR: those operations emit for rust without TransportEmissionNotModeled, this exclusion has an empty population and this row and the union's strip of those services delete together. Answering the channels as untruncated whole-stream lengths does NOT discharge it. Deleting the channels from the interface does NOT discharge it. Silently omitting the module from the union without naming this cause does NOT discharge it.", } diff --git a/dag/gunbc/rust_shell_stderr_capture_seed_growth.dag b/dag/gunbc/rust_shell_stderr_capture_seed_growth.dag new file mode 100644 index 00000000000..0d5da1ad8de --- /dev/null +++ b/dag/gunbc/rust_shell_stderr_capture_seed_growth.dag @@ -0,0 +1,71 @@ +module gunbc.rust_shell_stderr_capture_seed_growth + +import gunbc.roadmap_model { RoadmapNodeId } +import gunbc.seed_growth { SeedGrowthJustification } +import std.decl_ref { DeclarationRef, WholeDeclaration } + +// Seed-growth obligation for rust shell stderr capture (#13472), homed beside the +// obligation it declares rather than inside gunbc.seed_growth_admission, which owns +// the roster and the join. +// +// THE AUTHORIZER. Operator ruling 2026-10-06 (escalation msg_7853a1af, via +// silent-lark-156): "Admit, exact scope with seed-growth row". The admitted +// SeedFeatureCompletion is scoped to the rust shell handler realizing +// WitnessStderrCapturePolicy (stderr_truncated / stderr_total_bytes / +// stderr_retained_bytes) and the emit_host.rs tests that compile and run that +// fragment. This row records that ruling; it does not mint a second one. +// +// PURPOSE PASSES (gunbc.v1_maintenance_standing v1_seed_standing): the fixture-closure +// union must emit extdeps.gunbc WitnessBin.Run or the required floor stays red after +// #13437, which blocks the D2 self-host lane. The seed is the only rust shell emitter. +// +// THE FIVE REFUSED CLASSES. NewLanguageBehavior: no. NewCompatibilityObligation: no. +// NewEscapeHatchOrAdmissionRow: no — absent policy and Complete overflow refuse. +// SeedFeatureCompletion FIRES and is admitted on the ruling above. PublicSurfaceGrowth: +// the emit_host additions are cfg(test) discriminators, not a published CLI. +// +// Hand items below are the emit_host executing harness. The rust-item authority keeps +// the inline-module path, so every citation is +// v1_compiler.cli_run.emit_host.fixture_closure_union_tests. +// +// THE COMPLETE CEILING IS NOT A SECOND HAND MODULE (review 77633). Complete overflow +// reads v1_rt::read_host_budget_bytes / resolve_host_budget_join. Those functions +// live in rust_runtime_source (runtime_rust.dag rt_host_budget) because an emitted +// crate has no memory_governor. The stage0 v1_rt.rs mirror is generated +// (gunbc.generated_artifact / derived_generated_stage0_repo_paths), so it is not a +// SeedGrowthJustification hand item and must not be listed here. memory_governor +// wraps the same join; this change deletes the second composer rather than adding +// one. The 2026-10-06 ruling's handler scope includes that ceiling: current_boundary +// already names WitnessStderrCaptureCompleteBudgetExceeded with the admitted limit. +data rust_shell_stderr_capture_seed_growth_justification: SeedGrowthJustification = SeedGrowthJustification { + hand_authored_declarations: [ + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_MEMBER", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_UNMODELED_SIBLING", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_WITHOUT_POLICY", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_STRING_POLICY", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_OPTIONAL_POLICY", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_COLLECTION_POLICY", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "STDERR_CAPTURE_HOMONYM_POLICY", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "GUNBC_MODULE_REACH_MEMBER", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "declared_stderr_capture_channels_do_not_refuse_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "an_unmodeled_shell_channel_still_refuses_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "capture_channels_without_stderr_capture_input_refuse_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "capture_channels_refuse_unless_stderr_capture_is_the_required_scalar_policy", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "capture_channels_with_string_stderr_capture_refuse_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "capture_channels_with_optional_stderr_capture_refuse_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "capture_channels_with_collection_stderr_capture_refuse_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "capture_channels_with_homonym_stderr_capture_refuse_the_union", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "reaching_extdeps_gunbc_does_not_refuse_the_union_for_capture_channels", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "rustc_and_run_emitted_capture", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "emitted_bounded_tail_truncates_and_keeps_the_tail", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "emitted_bounded_tail_under_cap_is_complete", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "emitted_complete_over_budget_refuses", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "emitted_complete_under_budget_retains_all", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "emitted_stdin_and_long_stderr_does_not_deadlock", field: WholeDeclaration }, + DeclarationRef { module_path: "v1_compiler.cli_run.emit_host.fixture_closure_union_tests", decl_name: "emitted_absent_policy_refuses_before_spawn", field: WholeDeclaration } + ], + reason: "Operator ruling 2026-10-06 (msg_7853a1af): admit SeedFeatureCompletion, exact scope the rust shell handler realizing WitnessStderrCapturePolicy (stderr_truncated / stderr_total_bytes / stderr_retained_bytes) and its emit_host.rs tests. The rust shell handler must realize those channels so the fixture-closure union can emit WitnessBin.Run. Complete's admitted limit is the existing host-budget join on rust_runtime_source (v1_rt::read_host_budget_bytes), not a new seed-retained module and not a second composer in memory_governor. These host tests compile and run the emitted drain against a child process, and they refuse a name-only, String, optional, or collection stderr_capture at TransportEmissionNotModeled. A .contains() on the drain string is not a consumer (DESIGN section 5). They stay rust because the subject is the rust realization fragment the seed emits.", + owning_dissolution_lane: "v1-hand-queue-drain" as RoadmapNodeId, + trigger: "Delete these items when the v2-native emitter realizes the shell transport's stderr capture, a .dag witness compiles the emitted rust crate and runs the same six discriminators plus the typed-policy emit refusals without a host rustc harness, and WitnessBin.Run still emits. The union digest_hex control stays.", + current_boundary: "Bind and validate a required scalar stderr_capture: WitnessStderrCapturePolicy before spawn; start stdout and stderr drains immediately; write stdin concurrently; wait and join; project BoundedTail or return WitnessStderrCaptureCompleteBudgetExceeded with the measured total and the admitted limit from v1_rt::read_host_budget_bytes (rust_runtime_source / runtime_rust.dag rt_host_budget; memory_governor wraps the same join). No fallback tail length. Capture-accounting channels without that typed required scalar refuse at the rust emit diagnostic (one shared predicate) and, if that wall is skipped, as a pre-spawn return Err. Python and go still refuse those channels at emit. Unmodeled keys such as stderr_digest_hex still refuse." +} diff --git a/dag/gunbc/seed_growth_admission.dag b/dag/gunbc/seed_growth_admission.dag index 312cfb8a16d..33f69bd82d3 100644 --- a/dag/gunbc/seed_growth_admission.dag +++ b/dag/gunbc/seed_growth_admission.dag @@ -95,6 +95,7 @@ import gunbc.floor_checker_input_seed_growth { floor_checker_input_seed_growth_j import gunbc.data_eval_counters_seed_growth { data_eval_counters_seed_growth_justification } import gunbc.param_binder_seed_growth { param_binder_seed_growth_justification } import gunbc.process_lifecycle_seed_growth { process_lifecycle_seed_growth_justification } +import gunbc.rust_shell_stderr_capture_seed_growth { rust_shell_stderr_capture_seed_growth_justification } import gunbc.rust_item_host_observation { ItemDeclarationRefProjection, LiveHandRustObservation, @@ -381,7 +382,8 @@ fn seed_growth_justification_roster() -> List { match_arm_guard_seed_growth_justification, value_depth_seed_growth_justification, value_drop_binding_adaptation_seed_growth_justification, - process_lifecycle_seed_growth_justification + process_lifecycle_seed_growth_justification, + rust_shell_stderr_capture_seed_growth_justification ] } diff --git a/dag/gunbc/stderr_capture_gap_exclusion.dag b/dag/gunbc/stderr_capture_gap_exclusion.dag deleted file mode 100644 index 55cda509ef8..00000000000 --- a/dag/gunbc/stderr_capture_gap_exclusion.dag +++ /dev/null @@ -1,23 +0,0 @@ -module gunbc.stderr_capture_gap_exclusion - -// THE FIXTURE-CLOSURE UNION'S EXCLUDED OPERATION, as one record the seed compiles and evaluates. -// gunbc.rung_drop.fixture_closure_union_unmodeled_stderr_capture consumes this row. Authored names -// are the compiler's declaration identity for extdeps.gunbc / WitnessBin / Run (RungDrop.population -// stays List for every other drop; this carrier is the typed field that row cannot grow -// without migrating the ledger). - -type StderrCaptureGapExclusion { - drop_identity: String - declaring_module: String - service: String - operation_qualified: String - operation_bare: String -} - -data exclusion: StderrCaptureGapExclusion = StderrCaptureGapExclusion { - drop_identity: "gunbc.rung_drop.fixture_closure_union_unmodeled_stderr_capture", - declaring_module: "extdeps.gunbc", - service: "gunbc.WitnessBin", - operation_qualified: "gunbc.WitnessBin.Run", - operation_bare: "Run", -} diff --git a/dag/test/claim/transport_emission_not_modeled_witness_test.dag b/dag/test/claim/transport_emission_not_modeled_witness_test.dag index 56ab6bc2bf3..129884b024d 100644 --- a/dag/test/claim/transport_emission_not_modeled_witness_test.dag +++ b/dag/test/claim/transport_emission_not_modeled_witness_test.dag @@ -58,6 +58,16 @@ data unmodeled_output_key_source: String = "module tfx_key\nservice Fk \{\n ope data payload_verb_without_content_source: String = "module tfx_payload\nservice Fp \{\n operation Read \{\n input \{ path: String \}\n output \{ content: String from \"content\" \}\n readonly\n transport file \{ path: \"\{path\}\" \}\n \}\n operation Seal \{\n input \{ path: String \}\n output \{ success: Bool from \"write_success\" \}\n transport file \{ path: \"\{path\}\", verb: \"write_owner_only\" \}\n \}\n\}\n" +// The rust shell handler now realizes the declared stderr capture channels +// (truncation + both byte counts) from WitnessStderrCapturePolicy. The mixed +// row keeps the unmodeled-key RED beside that emission so a handler that +// answered every sibling, or a wall that poisoned every sibling, still fails. +data shell_stderr_capture_source: String = "module tfx_cap\nimport std.shell_stream_capture \{ WitnessStderrCapturePolicy, BoundedTail \}\nimport std.measure \{ byte_size \}\nservice Bin \{\n operation Run \{\n input \{\n bin_path: String\n stderr_capture: WitnessStderrCapturePolicy = BoundedTail \{ bytes: byte_size(count: 16) \}\n \}\n output \{\n success: Bool from \"exit_success\"\n stderr: String from \"stderr\"\n stderr_truncated: Bool from \"stderr_truncated\"\n stderr_total_bytes: Int from \"stderr_total_bytes\"\n stderr_retained_bytes: Int from \"stderr_retained_bytes\"\n \}\n transport shell \{ argv: [\"\{bin_path\}\"] \}\n \}\n\}\n" + +data shell_stderr_capture_beside_unmodeled_key_source: String = "module tfx_cap_mix\nimport std.shell_stream_capture \{ WitnessStderrCapturePolicy, BoundedTail \}\nimport std.measure \{ byte_size \}\nservice Bin \{\n operation Run \{\n input \{\n bin_path: String\n stderr_capture: WitnessStderrCapturePolicy = BoundedTail \{ bytes: byte_size(count: 16) \}\n \}\n output \{\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n stderr_total_bytes: Int from \"stderr_total_bytes\"\n stderr_retained_bytes: Int from \"stderr_retained_bytes\"\n \}\n transport shell \{ argv: [\"\{bin_path\}\"] \}\n \}\n operation Weird \{\n input \{ bin_path: String \}\n output \{ digest: String from \"stderr_digest_hex\" \}\n transport shell \{ argv: [\"\{bin_path\}\"] \}\n \}\n\}\n" + +data shell_stderr_capture_without_policy_input_source: String = "module tfx_cap_nop\nservice Bin \{\n operation Run \{\n input \{ bin_path: String \}\n output \{\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n \}\n transport shell \{ argv: [\"\{bin_path\}\"] \}\n \}\n\}\n" + fn not_modeled_blocking_count(source: String) -> Int { match compile_dag_diagnostic_census(source) { CensusObserved { rows: rows } => @@ -104,3 +114,19 @@ test fn w_unmodeled_output_channel_refuses_only_its_own_operation() -> Bool { test fn w_payload_verb_without_content_input_refuses_only_its_own_operation() -> Bool { not_modeled_blocking_count(source: payload_verb_without_content_source) == 1 } + +test fn w_declared_shell_stderr_capture_channels_emit() -> Bool { + not_modeled_blocking_count(source: shell_stderr_capture_source) == 0 +} + +test fn w_declared_shell_stderr_capture_module_is_clean_of_all_blocking_diagnostics() -> Bool { + blocking_diagnostic_count(source: shell_stderr_capture_source) == 0 +} + +test fn w_unmodeled_shell_channel_refuses_beside_realized_capture_channels() -> Bool { + not_modeled_blocking_count(source: shell_stderr_capture_beside_unmodeled_key_source) == 1 +} + +test fn w_capture_channel_without_stderr_capture_input_refuses() -> Bool { + not_modeled_blocking_count(source: shell_stderr_capture_without_policy_input_source) == 1 +} diff --git a/docs/design-rung-drops.md b/docs/design-rung-drops.md index 7d2c709ff07..8f9c248422e 100644 --- a/docs/design-rung-drops.md +++ b/docs/design-rung-drops.md @@ -142,9 +142,11 @@ an append to the fabric event log by a principal that is not a fleet writer: RUN per-module rust emit refusals of corpus modules reachable from required-floor fixture closures, outside the v1 seed closure: RUNG DROP, mechanically preventable -> mitigatable (lost as a passenger of gunbc#13037 (floor C1): compile_dag_rust_emit_check and compile_dag_diagnostic_census stopped rendering every closure module (RenderEveryModule -> v1_compiler.cli_run fixture_render_selection)). Population: every corpus module in the transitive import closure of a fixture source passed to compile_dag_rust_emit_check or compile_dag_diagnostic_census on the required floor, that is not a v1 seed module (no `// Source module:` header under src/v1/stage0/src) or is a seed mirror that the affected-set bound (v1_compiler.required_regen_host scope_selection) does not select on the run in question. Restored when: every corpus module reachable from a required-floor fixture closure is rendered at least once per required run through the rust emitter, and a per-module emit refusal refuses the floor. -### fixture-closure union rust emit of a shell operation whose modeled capture-accounting channels the rust target cannot realize — declared 2026-10-06 +### fixture-closure union rust emit of a shell operation whose modeled capture-accounting channels the rust target cannot realize — declared 2026-10-06 · RETIRED -fixture-closure union rust emit of a shell operation whose modeled capture-accounting channels the rust target cannot realize: RUNG DROP, mechanically preventable -> mitigatable (replacement staged: the rust emitted realization implementing the declared WitnessStderrCapturePolicy, the same next-rung trigger v1.compiler.emit shell_channel_realized_by_target already names). Population: exactly gunbc.stderr_capture_gap_exclusion.exclusion (consumed here as capture_gap_exclusion_population): gunbc.WitnessBin.Run in extdeps.gunbc. Output channels stderr_truncated, stderr_total_bytes, stderr_retained_bytes, each a TransportEmissionNotModeled ShellChannelNotRealizedByTarget for target rust. A later shell operation with the same capture-gap fact is NOT a member: the union still renders it and still refuses. The union compiles and evaluates that record; deleting either this module or the carrier refuses the seed compile. Restored when: THE CAPABILITY: the rust emitted shell realization implements the operation's declared WitnessStderrCapturePolicy, sufficient for v1.compiler.emit shell_channel_realized_by_target to return true for ShellChanStderrTruncated, ShellChanStderrTotalBytes and ShellChanStderrRetainedBytes. WHAT THAT MUST BE SUFFICIENT FOR: those operations emit for rust without TransportEmissionNotModeled, this exclusion has an empty population and this row and the union's strip of those services delete together. Answering the channels as untruncated whole-stream lengths does NOT discharge it. Deleting the channels from the interface does NOT discharge it. Silently omitting the module from the union without naming this cause does NOT discharge it. +**RETIRED — TRIGGER FIRED.** 2026-10-08 -- gunbc#13472. THE CAPABILITY, EXECUTED: v1.compiler.emit shell_channel_realized_by_target returns true for ShellChanStderrTruncated, ShellChanStderrTotalBytes and ShellChanStderrRetainedBytes on rust because emit_shell_call implements the declared WitnessStderrCapturePolicy. WitnessBin.Run emits without TransportEmissionNotModeled; the union strip of that service is deleted; this exclusion population is empty. DISCRIMINATING CONTROL, ENROLLED: v1_compiler.cli_run fixture_closure_union_controls admits a fixture whose closure reaches extdeps.gunbc, and a real emit error beside that reach still refuses as FixtureClosureUnionEmitRefused at module=efr_member. Answering the channels as untruncated whole-stream lengths did not discharge this. Deleting the channels from the interface did not discharge this. + +fixture-closure union rust emit of a shell operation whose modeled capture-accounting channels the rust target cannot realize: RUNG DROP, mechanically preventable -> mitigatable (replacement staged: the rust emitted realization implementing the declared WitnessStderrCapturePolicy, the same next-rung trigger v1.compiler.emit shell_channel_realized_by_target already names). Population: . Restored when: THE CAPABILITY: the rust emitted shell realization implements the operation's declared WitnessStderrCapturePolicy, sufficient for v1.compiler.emit shell_channel_realized_by_target to return true for ShellChanStderrTruncated, ShellChanStderrTotalBytes and ShellChanStderrRetainedBytes. WHAT THAT MUST BE SUFFICIENT FOR: those operations emit for rust without TransportEmissionNotModeled, this exclusion has an empty population and this row and the union's strip of those services delete together. Answering the channels as untruncated whole-stream lengths does NOT discharge it. Deleting the channels from the interface does NOT discharge it. Silently omitting the module from the union without naming this cause does NOT discharge it. ### new-witness eval-step cost gate over the one claim that runs the real fleet-converge build-job step list to hold the reset-observer dispatch admission's membership: it still executes, eval_steps stay recorded, a semantic red still blocks; only the eval-step cost-gate rung is lowered — declared 2026-09-30 diff --git a/src/v1/05_emit.dag b/src/v1/05_emit.dag index 3dc364c1280..a9e399702e8 100644 --- a/src/v1/05_emit.dag +++ b/src/v1/05_emit.dag @@ -34,13 +34,15 @@ import v1.std.core { is_file_transport, is_local_transport, TransportKind, RestTransport, ShellTransport, FileTransport, LocalTransport, classify_transport, transport_verb, transport_base_path, transport_has_auth, field_init_operation_modifier, operation_modifier_name, with_required_cardinality, tuple_type_name, find_child_named, - Connective, Conj, Disj, NoConnective, Arrow, CardOptional + Connective, Conj, Disj, NoConnective, Arrow, Required, CardOptional } -import v1.compiler.infer_env { TypeEnv, TypeBinding, authored_name, lookup_type_for, GlobalBareLookupState, empty_symbol_index } +import v1.compiler.infer_env { TypeEnv, TypeBinding, authored_name, lookup_type_for, GlobalBareLookupState, empty_symbol_index, type_reference_declaration_reading, TypeReferenceNamesDeclaration } import std.induction { InductiveField, SubValueUnknown } +import std.decl_ref { DeclarationRef, decl_ref, declaration_ref_eq } + import v1.compiler.infer_types { resolved_type, is_product_type, child_type_node, emit_map_has, node_is_collection, node_is_keyed_collection, node_is_element_collection, normalize_access_type_node, for_each_element_type_node, is_unit_like, is_declared_container_alias_spelling } import std.types { is_container_type } @@ -64,7 +66,8 @@ import v1.compiler.infer_lookup { lookup_func_sig } import v1.compiler.infer { InferScope, is_where_refinement_type, - build_params_scope, extend_scope, call_param_caller_labels + build_params_scope, extend_scope, call_param_caller_labels, + resolved_type_name } import v1.compiler.infer_emit_info { TypeSummary, EmitGraphInfo } @@ -1327,6 +1330,7 @@ type ShellResultField { type ShellEmissionRefusal = ShellOutputKeyNotModeled { key: String } | ShellChannelNotRealizedByTarget { key: String, target_name: String } + | ShellCapturePolicyInputAbsent { key: String } // The single authority for which declared output keys the shell realization can // answer. Read by the binding; there is no second list. @@ -1336,17 +1340,11 @@ type ShellEmissionRefusal // when no `from` key is authored -- so the two directions of the one procedure // (DESIGN section 4) read the same rule rather than agreeing by coincidence. // -// NOT MODELED, named rather than left to be inferred, because each names a fact -// the emitted realization cannot honestly produce: -// -// the byte-accounting and truncation channels -- stdout/stderr_total_bytes, -// _retained_bytes and _truncated. An operation declaring them also declares a -// capture policy (gunbc.WitnessBin.Run carries a WitnessStderrCapturePolicy -// input), and the emitted `std::process::Command` body implements no policy at -// all. Answering `false` and a raw length would not be a conservative default, -// it would assert that the declared policy ran and did not truncate. The -// next-rung trigger is the emitted realization implementing the declared -// capture policy; until then these refuse as a counted capability gap. +// The byte-accounting and truncation channels are modeled. The rust handler +// answers them from the declared WitnessStderrCapturePolicy (concurrent drain). +// Answering `false` and a raw length without running that policy is still +// forbidden; python and go refuse the three channels until they bind the same +// handler fact. // // the digest channels -- stdout/stderr_digest_hex, which the interpreter // answers and the emitted realization has no digest facility for. The @@ -1381,23 +1379,36 @@ fn shell_result_channel_key(c: ShellResultChannel) -> String { // WHETHER A TARGET CAN REALIZE A CHANNEL IS A FACT ABOUT THE TARGET, and keeping // it here rather than in the roster is the whole point of the split. // -// The three capture-accounting channels are REAL shell channels -- the interpreter -// answers all three from the declared WitnessStderrCapturePolicy -- and no emitted -// realization implements a capture policy at all. The emitted `Command` body -// captures whole streams, so `truncated` would always be false and total would -// always equal retained: not a conservative default but an assertion that the -// declared policy ran and did not truncate. -// -// NEXT-RUNG TRIGGER, at capability grain: the emitted realization implementing the -// declared capture policy, sufficient to answer truncation and both byte counts -// from the policy the operation declares. Until then every realizing target -// refuses these three, and the refusal is LOCATED AT THE FIELD. -fn shell_channel_realized_by_target(c: ShellResultChannel, target: RenderTarget) -> Bool { +// The three capture-accounting channels are REAL shell channels. The rust +// realization handler (v1.compiler.emit_rust emit_shell_call) implements the +// declared WitnessStderrCapturePolicy: concurrent drain, BoundedTail retains a +// tail, Complete retains within the host budget +// (`v1_rt::read_host_budget_bytes` over `v1_rt::resolve_host_budget_join`, +// the one composer `memory_governor::resolve_host_budget` also calls) or +// refuses. Python and go still realize no capture policy, so they refuse these +// three at field grain rather than answering false / raw length (that answer +// would assert the declared policy ran). +fn shell_channel_is_capture_accounting(c: ShellResultChannel) -> Bool { match c { - ShellChanStderrTruncated => false - ShellChanStderrTotalBytes => false - ShellChanStderrRetainedBytes => false - _ => target_renders_shell_transport(target: target) + ShellChanStderrTruncated => true + ShellChanStderrTotalBytes => true + ShellChanStderrRetainedBytes => true + _ => false + } +} + +fn shell_channel_realized_by_target(c: ShellResultChannel, target: RenderTarget) -> Bool { + if shell_channel_is_capture_accounting(c: c) { + target_is_rust(target: target) + } else { + target_renders_shell_transport(target: target) + } +} + +fn target_is_rust(target: RenderTarget) -> Bool { + match target { + Rust => true + _ => false } } @@ -1416,6 +1427,9 @@ fn shell_emission_refusal_fact(refusal: ShellEmissionRefusal) -> String { ShellChannelNotRealizedByTarget { key: k, target_name: tn } => concat("shell output channel '", k, "' is a modeled channel that target ", tn, " cannot realize -- the emitted realization implements no stderr capture policy, so answering it would assert that the declared policy ran") + ShellCapturePolicyInputAbsent { key: k } => + concat("shell output channel '", k, + "' is a capture-accounting channel and this operation does not declare a required scalar stderr_capture: WitnessStderrCapturePolicy -- answering it would apply a policy nobody declared") } } @@ -2577,6 +2591,31 @@ fn file_operation_has_content_input(op_node: Node, source_indices: Map DeclarationRef { + decl_ref(module_path: "std.shell_stream_capture", decl_name: "WitnessStderrCapturePolicy") +} + +fn param_type_is_stderr_capture_policy_declaration(p: Node, env: TypeEnv) -> Bool { + match type_reference_declaration_reading(n: param_node_type_expr(n: p), source_indices: env.source_indices, env: env) { + TypeReferenceNamesDeclaration { declaration: d } => + declaration_ref_eq(a: d, b: required_stderr_capture_policy_declaration()) + _ => false + } +} + +fn param_is_required_stderr_capture_policy(p: Node, env: TypeEnv) -> Bool { + let ty = param_node_type_expr(n: p) + param_node_name_at(n: p, source_indices: env.source_indices) == "stderr_capture" + && p.return_cardinality == Required + && ty.return_cardinality == Required + && !node_is_collection(n: ty, source_indices: env.source_indices) + && param_type_is_stderr_capture_policy_declaration(p: p, env: env) +} + +fn operation_declares_required_stderr_capture_policy(op_node: Node, env: TypeEnv) -> Bool { + op_node.params |> any(p => param_is_required_stderr_capture_policy(p: p, env: env)) +} + // The channels the interpreter's map_file_outputs answers, read back as the emitter's obligation. // Emission and interpretation are one decision procedure run in two directions (DESIGN §4), so // this list is the emitted side of that map and diverging from it is a behavioral fork. @@ -2816,7 +2855,24 @@ fn unmodeled_shell_transport_operation_diagnostics(tm: TypedModule, item: Node, }, module_name: module_name)] } Present { value: c } => - if shell_channel_realized_by_target(c: c, target: target) { [] } + if shell_channel_is_capture_accounting(c: c) + && target_is_rust(target: target) + && !operation_declares_required_stderr_capture_policy(op_node: op_node, env: env) + { + [make_error_node( + diagnostic: TransportEmissionNotModeled { + transport_kind: "shell", + service: service_name, + operation: authored_name(env: env, node: op_node), + declaring_module: module_name, + target: render_target_name(target: target), + missing_realization_fact: shell_emission_refusal_fact(refusal: ShellCapturePolicyInputAbsent { + key: shell_result_channel_key(c: c) + }), + span: ch.span + }, + module_name: module_name)] + } else if shell_channel_realized_by_target(c: c, target: target) { [] } else { [make_error_node( diagnostic: TransportEmissionNotModeled { transport_kind: "shell", diff --git a/src/v1/05_emit_rust.dag b/src/v1/05_emit_rust.dag index 87ac7976dcf..973db16d912 100644 --- a/src/v1/05_emit_rust.dag +++ b/src/v1/05_emit_rust.dag @@ -267,7 +267,8 @@ import v1.compiler.emit { ShellResultField, ShellResultChannel, ShellChanStdout, ShellChanStderr, ShellChanExitSuccess, ShellChanExitCode, ShellChanStdoutLines, ShellChanStderrTruncated, ShellChanStderrTotalBytes, ShellChanStderrRetainedBytes, - ShellChannelNotRealizedByTarget, shell_emission_refusal_fact, shell_result_channel_key, + ShellChannelNotRealizedByTarget, ShellCapturePolicyInputAbsent, shell_emission_refusal_fact, shell_result_channel_key, + shell_channel_is_capture_accounting, operation_declares_required_stderr_capture_policy, bind_operation_transport, transport_binding_refusal_fact, FileVerb, FileRead, FileWrite, FileWriteOwnerOnly, FileWriteCreateNew, FileWriteCreateNewWithMode, FileLinkCreateNew, FileDelete, FileList, FileResultField, FileResultChannel, FileChanSuccess, FileChanByteCount, @@ -16341,7 +16342,7 @@ fn emit_dry_run_branch_from_props(op_name: String, inferred: Node, mock_props: L fn emit_transport_call(bound: BoundOperation, op_name: String, registry: Map, depth: Int, service_item: Node, op_node: Node, source_indices: Map, shared_types: Set, env: TypeEnv) -> String { match bound { RestBound { transport: t } => emit_rest_call(op_name: op_name, transport: t, registry: registry, depth: depth, service_item: service_item, op_node: op_node, source_indices: source_indices, shared_types: shared_types, env: env) - ShellBound { transport: t, result_fields: rsf } => emit_shell_call(op_name: op_name, transport: t, registry: registry, depth: depth, result_fields: rsf, op_node: op_node, source_indices: source_indices) + ShellBound { transport: t, result_fields: rsf } => emit_shell_call(op_name: op_name, transport: t, registry: registry, depth: depth, result_fields: rsf, op_node: op_node, source_indices: source_indices, env: env) FileBound { verb: v, path_template: pt, result_fields: fs } => emit_file_call(verb: v, path_template: pt, result_fields: fs, source_indices: source_indices) LocalBound => emit_local_call(op_name: op_name) @@ -16871,7 +16872,7 @@ fn emit_exit_code_handling(op_node: Node, result_fields: List, let has_nonzero = exit_props |> any(p => field_init_node_name_at(n: p, source_indices: source_indices) == "exit_nonzero") let exit_arms = exit_props |> map(p => emit_exit_arm(prop: p, result_fields: result_fields, source_indices: source_indices)) let default_arm = if has_nonzero { "" } - else { concat("\n _ => { ", shell_stderr_binding_line, " ", emit_rust_boxed_error_return(message_expr: "stderr"), " },") } + else { concat("\n _ => { ", shell_error_stderr_binding(result_fields: result_fields), " ", emit_rust_boxed_error_return(message_expr: "stderr"), " },") } concat( "let exit_code = output.status.code().unwrap_or(-1);\n", "match exit_code {\n", @@ -16890,7 +16891,7 @@ fn emit_exit_arm(prop: Node, result_fields: List, source_indic if is_success { concat(" ", pattern, " => { ", emit_shell_return(result_fields: result_fields), " },") } else { - concat(" ", pattern, " => { ", shell_stderr_binding_line, " ", emit_rust_boxed_error_return(message_expr: "stderr"), " },") + concat(" ", pattern, " => { ", shell_error_stderr_binding(result_fields: result_fields), " ", emit_rust_boxed_error_return(message_expr: "stderr"), " },") } } @@ -16900,7 +16901,10 @@ fn emit_exit_arm(prop: Node, result_fields: List, source_indic // replace one loud stop with another whose red this route cannot adjudicate (an emitted panic // COMPILES, so the fixture pairs beside this change would report it green). Only the LATER elements // admit both shapes; see shell_argv_element_is_word_list. -fn emit_shell_call(op_name: String, transport: Node, registry: Map, depth: Int, result_fields: List, op_node: Node, source_indices: Map) -> String { +// Capture order: bind/validate the authored policy BEFORE spawn; start both drains +// immediately; write stdin concurrently; join; then project BoundedTail or refuse +// Complete overflow. +fn emit_shell_call(op_name: String, transport: Node, registry: Map, depth: Int, result_fields: List, op_node: Node, source_indices: Map, env: TypeEnv) -> String { let argv = transport.children let optional_params = op_node.params |> filter(p => param_node_type_expr(n: p).return_cardinality == CardOptional) @@ -16912,7 +16916,8 @@ fn emit_shell_call(op_name: String, transport: Node, registry: Map true Absent => false } - let let_kw = if has_stdin { "let mut output" } else { "let output" } + let needs_capture = shell_needs_capture_accounting(result_fields: result_fields) + let let_kw = if has_stdin || needs_capture { "let mut output" } else { "let output" } let cmd_line = if argv |> count > 0 { match argv |> first { Present { value: first_arg } => concat(let_kw, " = std::process::Command::new(", @@ -16938,7 +16943,24 @@ fn emit_shell_call(op_name: String, transport: Node, registry: Map emit_ident(name: expr_var_name_at(texpr: stdin_expr, source_indices: source_indices), target: Rust) + _ => emit_simple_expr(expr: stdin_expr, target: Rust, source_indices: source_indices) + } + concat("let mut stdin_pipe = output.stdin.take();\nlet __stdin_thread = std::thread::spawn(move || -> std::io::Result<()> {\n use std::io::Write;\n if let Some(mut stdin) = stdin_pipe {\n stdin.write_all(", stdin_var, ".as_bytes())?;\n }\n Ok(())\n});\n") + } else { + "let mut stdin_pipe = output.stdin.take();\nlet __stdin_thread = std::thread::spawn(move || -> std::io::Result<()> {\n drop(stdin_pipe);\n Ok(())\n});\n" + } + let capture_lines = concat(shell_capture_drain_start, stdin_thread, shell_capture_join_project) + concat([policy_bind, cmd_line], arg_lines, env_lines, [wd_line, spawn_line, spawn_exec, capture_lines, return_line]) |> join(separator: "\n") + } else if has_stdin { let stdin_expr = transport_stdin(t: transport, source_indices: source_indices).value let stdin_var = match stdin_expr.expr_data { ExprVar { binding_kind: _ } => emit_ident(name: expr_var_name_at(texpr: stdin_expr, source_indices: source_indices), target: Rust) @@ -16949,15 +16971,11 @@ fn emit_shell_call(op_name: String, transport: Node, registry: Map join(separator: "\n") + concat([cmd_line], arg_lines, env_lines, [wd_line, spawn_line, spawn_exec, write_block, wait_line, check_line, return_line]) |> join(separator: "\n") } else { let output_line = " .output()?;" let check_line = "let stdout = String::from_utf8_lossy(&output.stdout).to_string();" - let return_line = emit_exit_code_handling(op_node: op_node, result_fields: result_fields, source_indices: source_indices) - let all_lines = concat([cmd_line], arg_lines, env_lines, [wd_line, output_line, check_line, return_line]) - all_lines |> join(separator: "\n") + concat([cmd_line], arg_lines, env_lines, [wd_line, output_line, check_line, return_line]) |> join(separator: "\n") } } @@ -17065,6 +17083,48 @@ fn emit_shell_argv_element(arg: Node, optional_params: Map, source // error. data shell_stderr_binding_line: String = "let stderr = String::from_utf8_lossy(&output.stderr).to_string();" +fn shell_needs_capture_accounting(result_fields: List) -> Bool { + result_fields |> any(f => shell_channel_is_capture_accounting(c: f.channel)) +} + +fn shell_error_stderr_binding(result_fields: List) -> String { + if shell_needs_capture_accounting(result_fields: result_fields) { "" } else { concat(shell_stderr_binding_line, " ") } +} + +fn emit_shell_stderr_policy_binding(op_node: Node, env: TypeEnv) -> String { + if operation_declares_required_stderr_capture_policy(op_node: op_node, env: env) { + concat( + "let (__stderr_complete_limit, __stderr_complete_source): (Option, String) = match &*stderr_capture {\n", + " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::Complete => {\n", + " match v1_rt::read_host_budget_bytes() {\n", + " (Some(n), source) => (Some(n as usize), source),\n", + " (None, source) => return Err(format!(\"WitnessStderrCaptureCompleteBudgetUnreadable: Complete stderr capture requires the active host budget authority ({})\", source).into()),\n", + " }\n", + " }\n", + " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::BoundedTail { bytes } => {\n", + " let n = crate::std_measure::byte_size_count(bytes.clone());\n", + " if n < 0 { return Err(\"WitnessStderrCapturePolicy.BoundedTail bytes must be non-negative\".into()); }\n", + " (None, String::new())\n", + " }\n", + "};\n", + "let __stderr_tail_bytes: usize = match &*stderr_capture {\n", + " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::BoundedTail { bytes } => crate::std_measure::byte_size_count(bytes.clone()) as usize,\n", + " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::Complete => 0,\n", + "};\n" + ) + } else { + concat( + "return Err(\"", + shell_emission_refusal_fact(refusal: ShellCapturePolicyInputAbsent { key: "stderr_truncated" }), + "\".into());\n" + ) + } +} + +data shell_capture_drain_start: String = "let mut stdout_pipe = output.stdout.take();\nlet mut stderr_pipe = output.stderr.take();\nlet stdout_thread = std::thread::spawn(move || -> std::io::Result> {\n let mut buf = Vec::new();\n if let Some(ref mut reader) = stdout_pipe {\n std::io::Read::read_to_end(reader, &mut buf)?;\n }\n Ok(buf)\n});\nlet stderr_thread = std::thread::spawn(move || -> std::io::Result<(Vec, u64, bool)> {\n let mut reader = match stderr_pipe {\n Some(r) => r,\n None => return Ok((Vec::new(), 0, false)),\n };\n let mut chunk = [0u8; 65536];\n let mut total: u64 = 0;\n if let Some(max_bytes) = __stderr_complete_limit {\n let mut retained = Vec::new();\n loop {\n let n = std::io::Read::read(&mut reader, &mut chunk)?;\n if n == 0 { break; }\n total += n as u64;\n if total <= max_bytes as u64 { retained.extend_from_slice(&chunk[..n]); }\n }\n let truncated = total > max_bytes as u64;\n return Ok((if truncated { Vec::new() } else { retained }, total, truncated));\n }\n let cap = __stderr_tail_bytes;\n let mut ring = Vec::with_capacity(cap);\n let mut start = 0usize;\n loop {\n let n = std::io::Read::read(&mut reader, &mut chunk)?;\n if n == 0 { break; }\n total += n as u64;\n for &b in &chunk[..n] {\n if cap == 0 { continue; }\n if ring.len() < cap {\n ring.push(b);\n } else {\n ring[start] = b;\n start = (start + 1) % cap;\n }\n }\n }\n let retained = if ring.len() < cap || cap == 0 {\n ring\n } else {\n let mut out = Vec::with_capacity(cap);\n for i in 0..cap { out.push(ring[(start + i) % cap]); }\n out\n };\n let retained_len = retained.len() as u64;\n Ok((retained, total, total > retained_len))\n});\n" + +data shell_capture_join_project: String = "let status = output.wait()?;\n__stdin_thread.join().map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, \"stdin write thread panicked\"))??;\nlet stdout_bytes = stdout_thread.join().map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, \"stdout drain thread panicked\"))??;\nlet (stderr_bytes, stderr_total_u64, stderr_truncated) = stderr_thread.join().map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, \"stderr drain thread panicked\"))??;\nif let Some(max_bytes) = __stderr_complete_limit {\n if stderr_total_u64 > max_bytes as u64 {\n return Err(format!(\"WitnessStderrCaptureCompleteBudgetExceeded: stderr total {} exceeds host budget {} ({})\", stderr_total_u64, max_bytes, __stderr_complete_source).into());\n }\n}\nlet stderr_total_bytes = stderr_total_u64 as i64;\nlet stderr_retained_bytes = stderr_bytes.len() as i64;\nlet stdout = String::from_utf8_lossy(&stdout_bytes).to_string();\nlet stderr = String::from_utf8_lossy(&stderr_bytes).trim_end().to_string();\nlet output = std::process::Output { status, stdout: stdout_bytes, stderr: stderr_bytes };\n" + // Answers the declared fields in declared order, from the projection the binding // resolved, as the method's SUCCESS value. // @@ -17103,11 +17163,14 @@ fn shell_projection_value(result_fields: List) -> String { // `stderr` is captured only when a declared field asks for it, so an operation // that does not is not handed an unused binding. fn shell_stderr_prelude(result_fields: List) -> String { - let needs_stderr = result_fields |> any(f => match f.channel { - ShellChanStderr => true - _ => false - }) - if needs_stderr { concat(shell_stderr_binding_line, "\n") } else { "" } + if shell_needs_capture_accounting(result_fields: result_fields) { "" } + else { + let needs_stderr = result_fields |> any(f => match f.channel { + ShellChanStderr => true + _ => false + }) + if needs_stderr { concat(shell_stderr_binding_line, "\n") } else { "" } + } } // TOTAL BY CONSTRUCTION over the channels the binding resolves. The `stdout` @@ -17125,9 +17188,9 @@ fn emit_shell_channel_expr(channel: ShellResultChannel, is_optional: Bool) -> St ShellChanExitCode => "(output.status.code().unwrap_or(-1) as i64)" ShellChanStdoutLines => rust_shared_wrap_ctor(inner_expr: "stdout.lines().filter(|l| !l.is_empty()).map(|l| l.trim().to_string()).collect()") - ShellChanStderrTruncated => emit_unrealizable_shell_channel(channel: channel) - ShellChanStderrTotalBytes => emit_unrealizable_shell_channel(channel: channel) - ShellChanStderrRetainedBytes => emit_unrealizable_shell_channel(channel: channel) + ShellChanStderrTruncated => "stderr_truncated" + ShellChanStderrTotalBytes => "stderr_total_bytes" + ShellChanStderrRetainedBytes => "stderr_retained_bytes" } if is_optional { concat("Some(", base, ")") } else { base } } diff --git a/src/v1/runtime_rust.dag b/src/v1/runtime_rust.dag index 5129153804f..dfee9221126 100644 --- a/src/v1/runtime_rust.dag +++ b/src/v1/runtime_rust.dag @@ -1303,7 +1303,201 @@ fn rust_runtime_source() -> String { rt_filesystem(), rt_checked_int_ops(), rt_realization_measurement(), - rt_accelerator_demo_kernel()) + rt_accelerator_demo_kernel(), + rt_host_budget()) +} + +fn rt_host_budget() -> String { + concat( + "/// The one host-budget precedence. `read_host_budget_bytes` and\n", + "/// `memory_governor::resolve_host_budget` both call `resolve_host_budget_join`.\n", + "#[derive(Clone, Debug, PartialEq, Eq)]\n", + "pub enum HostBudgetJoinSource \{\n", + " CgroupMemoryHigh \{ cgroup_dir: String \},\n", + " CgroupMemoryMax \{ cgroup_dir: String \},\n", + " CgroupV1HierarchicalMemoryLimit \{ cgroup_dir: String \},\n", + " DarwinPhysicalMemory,\n", + "\}\n\n", + "impl HostBudgetJoinSource \{\n", + " pub fn label(&self) -> String \{\n", + " match self \{\n", + " HostBudgetJoinSource::CgroupMemoryHigh \{ cgroup_dir \} => format!(\"cgroup memory.high (\{\})\", cgroup_dir),\n", + " HostBudgetJoinSource::CgroupMemoryMax \{ cgroup_dir \} => format!(\"cgroup memory.max (\{\})\", cgroup_dir),\n", + " HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit \{ cgroup_dir \} => format!(\"cgroup v1 memory.stat hierarchical_memory_limit (\{\})\", cgroup_dir),\n", + " HostBudgetJoinSource::DarwinPhysicalMemory => \"sysctl hw.memsize\".to_string(),\n", + " \}\n", + " \}\n", + "\}\n\n", + "#[derive(Clone, Debug, PartialEq, Eq)]\n", + "pub enum HostBudgetJoin \{\n", + " Resolved \{ effective_bytes: u64, requested_bytes: Option, source: HostBudgetJoinSource, observed_bytes: u64 \},\n", + " DeclaredUnverified \{ requested_bytes: u64, reason: String \},\n", + " Unreadable \{ reason: String \},\n", + "\}\n\n", + "impl HostBudgetJoin \{\n", + " pub fn bytes(&self) -> Option \{\n", + " match self \{\n", + " HostBudgetJoin::Resolved \{ effective_bytes, .. \} => Some(*effective_bytes),\n", + " HostBudgetJoin::DeclaredUnverified \{ requested_bytes, .. \} => Some(*requested_bytes),\n", + " HostBudgetJoin::Unreadable \{ .. \} => None,\n", + " \}\n", + " \}\n", + " pub fn label(&self) -> String \{\n", + " match self \{\n", + " HostBudgetJoin::Resolved \{ effective_bytes, requested_bytes, source, observed_bytes \} => match requested_bytes \{\n", + " Some(requested) => format!(\"effective planning minimum \{\} bytes (env request \{\}; observed \{\}=\{\} bytes)\", effective_bytes, requested, source.label(), observed_bytes),\n", + " None => source.label(),\n", + " \},\n", + " HostBudgetJoin::Unreadable \{ reason \} => format!(\"unreadable: \{\}\", reason),\n", + " HostBudgetJoin::DeclaredUnverified \{ requested_bytes, reason \} => format!(\"declared-unverified: env GUNBC_MEMORY_BUDGET_BYTES=\{\}; \{\}\", requested_bytes, reason),\n", + " \}\n", + " \}\n", + "\}\n\n", + "pub fn resolve_host_budget_join(env_override: Option, cgroup_high: Option<(String, u64)>, cgroup_max: Option<(String, u64)>, cgroup_v1_limit: Option<(String, HostBudgetCgroupV1)>, darwin_physical: Option) -> HostBudgetJoin \{\n", + " let cgroup_v1_limit = match cgroup_v1_limit \{\n", + " Some((dir, HostBudgetCgroupV1::Unparseable(body))) => \{\n", + " return HostBudgetJoin::Unreadable \{ reason: format!(\"cgroup v1 memory hierarchy at \{\} holds this process but its hierarchical_memory_limit is unreadable (\{\}); a bound that may be the tightest cannot be replaced by another reading\", dir, body) \};\n", + " \}\n", + " Some((dir, HostBudgetCgroupV1::Limited(bytes))) => Some((dir, bytes)),\n", + " Some((_, HostBudgetCgroupV1::Unlimited)) | None => None,\n", + " \};\n", + " let observation = [\n", + " cgroup_high.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupMemoryHigh \{ cgroup_dir \}, b)),\n", + " cgroup_max.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupMemoryMax \{ cgroup_dir \}, b)),\n", + " cgroup_v1_limit.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit \{ cgroup_dir \}, b)),\n", + " ].into_iter().flatten().fold(None::<(HostBudgetJoinSource, u64)>, |best, cand| match best \{ Some(cur) if cur.1 <= cand.1 => Some(cur), _ => Some(cand) \});\n", + " if let Some((source, observed_bytes)) = observation \{\n", + " return HostBudgetJoin::Resolved \{ effective_bytes: env_override.map(|requested| requested.min(observed_bytes)).unwrap_or(observed_bytes), requested_bytes: env_override, source, observed_bytes \};\n", + " \}\n", + " if let Some(bytes) = darwin_physical \{\n", + " return HostBudgetJoin::Resolved \{ effective_bytes: env_override.map(|requested| requested.min(bytes)).unwrap_or(bytes), requested_bytes: env_override, source: HostBudgetJoinSource::DarwinPhysicalMemory, observed_bytes: bytes \};\n", + " \}\n", + " if let Some(requested_bytes) = env_override \{\n", + " return HostBudgetJoin::DeclaredUnverified \{ requested_bytes, reason: \"no observed private memory.high, memory.max or v1 hierarchical_memory_limit verifies the executor allowance; the declaration is a planning request, not an enforced process limit\".to_string() \};\n", + " \}\n", + " HostBudgetJoin::Unreadable \{ reason: format!(\"no cgroup memory.high, memory.max or v1 hierarchical_memory_limit binds this process and GUNBC_MEMORY_BUDGET_BYTES cannot verify one (target_os=\{\}), so the planning allowance is UNKNOWN. Refusing rather than admitting against the widest signal available: a host-shared reading is a number about the MACHINE, not about this slot, and admitting against one is the rc=137 SIGKILL this arm exists to prevent (BuildBuddy receipt 2026-08-30, gunbc.host_budget_source host_budget_source_seed_mirror_disposition). The executor must expose an enforceable limit; GUNBC_MEMORY_BUDGET_BYTES may only request a lower planning ceiling.\", std::env::consts::OS) \}\n", + "\}\n\n", + "pub fn read_host_budget_bytes() -> (Option, String) \{\n", + " let join = resolve_host_budget_join(\n", + " std::env::var(\"GUNBC_MEMORY_BUDGET_BYTES\").ok().and_then(|s| s.trim().parse::().ok()),\n", + " host_budget_tightest_cgroup(\"memory.high\"),\n", + " host_budget_tightest_cgroup(\"memory.max\"),\n", + " host_budget_cgroup_v1(),\n", + " host_budget_darwin_physical(),\n", + " );\n", + " (join.bytes(), join.label())\n", + "\}\n\n", + "#[derive(Clone)]\n", + "pub enum HostBudgetCgroupV1 \{\n", + " Limited(u64),\n", + " Unlimited,\n", + " Unparseable(String),\n", + "\}\n\n", + "pub fn host_budget_tightest_cgroup_under(self_cg: &str, root: &std::path::Path, limit_file: &str) -> Option<(String, u64)> \{\n", + " let rel = self_cg\n", + " .lines()\n", + " .find_map(|l| l.strip_prefix(\"0::\"))\n", + " .map(|p| p.trim().trim_start_matches('/').to_string())?;\n", + " let mut dir = root.join(&rel);\n", + " let mut best: Option<(u64, std::path::PathBuf)> = None;\n", + " loop \{\n", + " if let Ok(s) = std::fs::read_to_string(dir.join(limit_file)) \{\n", + " let s = s.trim();\n", + " if s != \"max\" \{\n", + " if let Ok(v) = s.parse::() \{\n", + " let take = best.as_ref().map(|(cur, _)| v < *cur).unwrap_or(true);\n", + " if take \{\n", + " best = Some((v, dir.clone()));\n", + " \}\n", + " \}\n", + " \}\n", + " \}\n", + " if dir == root || !dir.pop() \{\n", + " break;\n", + " \}\n", + " \}\n", + " best.map(|(v, d)| (d.display().to_string(), v))\n", + "\}\n\n", + "pub fn host_budget_tightest_cgroup(limit_file: &str) -> Option<(String, u64)> \{\n", + " let self_cg = std::fs::read_to_string(\"/proc/self/cgroup\").ok()?;\n", + " host_budget_tightest_cgroup_under(&self_cg, std::path::Path::new(\"/sys/fs/cgroup\"), limit_file)\n", + "\}\n\n", + "pub fn host_budget_cgroup_v1_unlimited_bytes(page_size: u64) -> u64 \{\n", + " (i64::MAX as u64 / page_size) * page_size\n", + "\}\n\n", + "pub fn host_budget_cgroup_v1() -> Option<(String, HostBudgetCgroupV1)> \{\n", + " let self_cg = std::fs::read_to_string(\"/proc/self/cgroup\").ok()?;\n", + " let mountinfo = std::fs::read_to_string(\"/proc/self/mountinfo\").ok()?;\n", + " let page_size = unsafe \{ libc::sysconf(libc::_SC_PAGESIZE) \};\n", + " if page_size <= 0 \{\n", + " return Some((\n", + " \"/proc/self/mountinfo\".to_string(),\n", + " HostBudgetCgroupV1::Unparseable(\"sysconf(_SC_PAGESIZE) unreadable\".to_string()),\n", + " ));\n", + " \}\n", + " host_budget_cgroup_v1_under(std::path::Path::new(\"/\"), &self_cg, &mountinfo, page_size as u64)\n", + "\}\n\n", + "pub fn host_budget_cgroup_v1_under(fs_root: &std::path::Path, self_cg: &str, mountinfo: &str, page_size: u64) -> Option<(String, HostBudgetCgroupV1)> \{\n", + " let dir = host_budget_cgroup_v1_memory_dir(self_cg, mountinfo)?;\n", + " let dir_path = std::path::Path::new(&dir);\n", + " let joined = fs_root.join(dir_path.strip_prefix(\"/\").unwrap_or(dir_path));\n", + " let value = match std::fs::read_to_string(joined.join(\"memory.stat\")) \{\n", + " Ok(stat) => host_budget_cgroup_v1_from_stat(&stat, page_size),\n", + " Err(e) => HostBudgetCgroupV1::Unparseable(format!(\"memory.stat: \{\}\", e)),\n", + " \};\n", + " Some((joined.display().to_string(), value))\n", + "\}\n\n", + "pub fn host_budget_cgroup_v1_memory_dir(self_cg: &str, mountinfo: &str) -> Option \{\n", + " let (mount_root, mount_point) = mountinfo.lines().find_map(|line| \{\n", + " let fields: Vec<&str> = line.split(' ').collect();\n", + " let dash = fields.iter().position(|f| *f == \"-\")?;\n", + " let fstype = fields.get(dash + 1)?;\n", + " let super_opts = fields.get(dash + 3)?;\n", + " if *fstype == \"cgroup\" && super_opts.split(',').any(|o| o == \"memory\") \{\n", + " Some((fields.get(3)?.to_string(), fields.get(4)?.to_string()))\n", + " \} else \{\n", + " None\n", + " \}\n", + " \})?;\n", + " let path = self_cg.lines().find_map(|l| \{\n", + " let mut parts = l.splitn(3, ':');\n", + " let (_id, controllers, path) = (parts.next()?, parts.next()?, parts.next()?);\n", + " controllers.split(',').any(|c| c == \"memory\").then(|| path.trim().to_string())\n", + " \})?;\n", + " let rel = if mount_root == \"/\" \{\n", + " path.as_str()\n", + " \} else \{\n", + " let rest = path.strip_prefix(mount_root.as_str())?;\n", + " if !(rest.is_empty() || rest.starts_with('/')) \{ return None; \}\n", + " rest\n", + " \};\n", + " Some(std::path::Path::new(&mount_point).join(rel.trim_start_matches('/')).display().to_string())\n", + "\}\n\n", + "pub fn host_budget_cgroup_v1_from_stat(memory_stat: &str, page_size: u64) -> HostBudgetCgroupV1 \{\n", + " let hits: std::vec::Vec<&str> = memory_stat.lines().filter_map(|l| l.trim().strip_prefix(\"hierarchical_memory_limit \")).collect();\n", + " let [body] = hits.as_slice() else \{ return HostBudgetCgroupV1::Unparseable(memory_stat.to_string()); \};\n", + " let unlimited = host_budget_cgroup_v1_unlimited_bytes(page_size);\n", + " match body.trim().parse::() \{\n", + " Ok(n) if n < 0 => HostBudgetCgroupV1::Unparseable(body.to_string()),\n", + " Ok(n) if n >= unlimited as i128 => HostBudgetCgroupV1::Unlimited,\n", + " Ok(n) => HostBudgetCgroupV1::Limited(n as u64),\n", + " Err(_) => HostBudgetCgroupV1::Unparseable(body.to_string()),\n", + " \}\n", + "\}\n\n", + "/// Darwin `hw.memsize` via `sysctlbyname`. Authority: `extdeps.darwin.sysctl` `HwMemsize`.\n", + "pub fn host_budget_darwin_physical() -> Option \{\n", + " #[cfg(target_os = \"macos\")]\n", + " \{\n", + " let name = std::ffi::CStr::from_bytes_with_nul(b\"hw.memsize\\0\").ok()?;\n", + " let mut value: u64 = 0;\n", + " let mut len: libc::size_t = std::mem::size_of::() as libc::size_t;\n", + " let rc = unsafe \{ libc::sysctlbyname(name.as_ptr(), (&mut value as *mut u64).cast::(), &mut len, std::ptr::null_mut(), 0) \};\n", + " if rc == 0 && value > 0 \{ Some(value) \} else \{ None \}\n", + " \}\n", + " #[cfg(not(target_os = \"macos\"))]\n", + " \{ None \}\n", + "\}\n" + ) } fn rt_accelerator_demo_kernel() -> String { diff --git a/src/v1/stage0/src/cli_run/emit_host.rs b/src/v1/stage0/src/cli_run/emit_host.rs index 915ca8dfec5..6182963a91d 100644 --- a/src/v1/stage0/src/cli_run/emit_host.rs +++ b/src/v1/stage0/src/cli_run/emit_host.rs @@ -51,7 +51,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::rc::Rc; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use crate::coproduct_reflection::{decl_facts_corpus_walk, DeclFactRaw}; use crate::module_path_index::{ @@ -3123,10 +3123,9 @@ pub(crate) fn fixture_closure_union_digest(members: &BTreeMap) - /// is the compile's list followed by the emitter's (`emit_resolved_for_target_selected`), so /// the suffix past the compile's length is exactly what the emitter added. /// -/// Unmodeled rust stderr-capture channels (`gunbc.rung_drop` -/// `fixture_closure_union_unmodeled_stderr_capture`) are a typed exclusion, not this refusal: -/// those services are stripped from the emit graph and listed on the observation. A sibling -/// unmodeled key or any other emit error still refuses. +/// Rust realizes the three capture-accounting channels. An unmodeled sibling key or any +/// other emit error still refuses. The retired drop +/// `fixture_closure_union_unmodeled_stderr_capture` no longer strips a service. pub(crate) fn fixture_closure_union_emit_receipt( union: &FixtureClosureUnion, ) -> Result { @@ -3198,7 +3197,6 @@ pub(crate) fn fixture_closure_union_emit_receipt( ), )); } - let (resolved, excluded) = union_emit_graph_excluding_unmodeled_stderr_capture(resolved); let compile_diagnostics = resolved.diagnostics.len(); let rendered = v1_compiler_compile::emit_resolved_for_target( resolved, @@ -3229,369 +3227,10 @@ pub(crate) fn fixture_closure_union_emit_receipt( digest, files: rendered.files.len(), emit_diagnostics: emitted.len(), - excluded, + excluded: Vec::new(), }) } -/// Deleting either the drop row or its typed carrier fails this compile. The union compiles -/// the carrier through `compile_to_resolved` and evaluates `exclusion` (review 77125). -const STDERR_CAPTURE_GAP_CARRIER_SOURCE: &str = - include_str!("../../../../../dag/gunbc/stderr_capture_gap_exclusion.dag"); -const STDERR_CAPTURE_POLICY_DROP_SOURCE: &str = include_str!( - "../../../../../dag/gunbc/rung_drop/fixture_closure_union_unmodeled_stderr_capture.dag" -); - -struct CaptureGapExclusion { - drop_identity: String, - declaring_module: String, - service: String, - operation_qualified: String, - operation_bare: String, -} - -fn record_field_string( - ctx: &crate::v1_interpreter::InterpContext, - fields: &[(crate::v1_interpreter::Symbol, crate::v1_interpreter::Value)], - name: &str, -) -> Result { - use crate::v1_interpreter::Value; - match fields - .iter() - .find(|(sym, _)| ctx.sym_eq(*sym, name)) - .map(|(_, v)| v) - { - Some(Value::Str(s)) => Ok(s.to_string()), - Some(other) => Err(format!( - "cause=CaptureGapExclusionFieldNotString field={name} value={other:?}" - )), - None => Err(format!( - "cause=CaptureGapExclusionFieldMissing field={name}" - )), - } -} - -fn load_capture_gap_exclusion() -> Result { - let _drop_row_tether = STDERR_CAPTURE_POLICY_DROP_SOURCE; - let files = vec![Rc::new(v1_compiler_compile::SourceFile { - path: "dag/gunbc/stderr_capture_gap_exclusion.dag".to_string(), - content: STDERR_CAPTURE_GAP_CARRIER_SOURCE.to_string(), - })]; - let resolved = v1_compiler_compile::compile_to_resolved(Rc::new(files.into())); - let located = |d: &Rc| { - format!( - "module={} reason=`{}`", - d.module_name, - crate::v1_std_core::diagnostic_to_message(d.diagnostic.clone()) - ) - }; - if v1_compiler_compile::emittable_graph(resolved.clone()).is_none() { - let blocking: Vec = resolved - .diagnostics - .iter() - .filter(|d| { - crate::v1_std_core::is_interpreter_blocking_diagnostic(d.diagnostic.clone()) - }) - .map(located) - .collect(); - return Err(format!( - "cause=CaptureGapExclusionUncompilable drop_row_bytes={} diagnostics={}", - _drop_row_tether.len(), - blocking.join(" | ") - )); - } - let Some(graph) = resolved.graph.clone() else { - return Err(format!( - "cause=CaptureGapExclusionUncompilable drop_row_bytes={} -- resolved graph is none", - _drop_row_tether.len() - )); - }; - let ctx = make_eval_context( - &graph, - resolved.source_indices.clone(), - crate::v1_interpreter::ExecutionMode::Hermetic, - ); - let val = crate::v1_interpreter::with_active_context(&ctx, || { - match crate::v1_interpreter::eval_data_item_value(&ctx, "exclusion") { - Ok(Some(v)) => Ok(Some(v)), - Ok(None) => crate::v1_interpreter::eval_data_item_value( - &ctx, - "gunbc.stderr_capture_gap_exclusion.exclusion", - ), - Err(e) => Err(e), - } - }) - .map_err(|e| format!("cause=CaptureGapExclusionEvalFailed error={e}"))? - .ok_or_else(|| "cause=CaptureGapExclusionDataMissing name=exclusion".to_string())?; - let crate::v1_interpreter::Value::Record { - ref fields, - type_name, - } = val - else { - return Err(format!("cause=CaptureGapExclusionNotRecord value={val:?}")); - }; - if !ctx.sym_eq(type_name, "StderrCaptureGapExclusion") - && !ctx.sym_eq( - type_name, - "gunbc.stderr_capture_gap_exclusion.StderrCaptureGapExclusion", - ) - { - return Err(format!( - "cause=CaptureGapExclusionUnexpectedType type={}", - ctx.resolve(type_name) - )); - } - Ok(CaptureGapExclusion { - drop_identity: record_field_string(&ctx, fields.as_slice(), "drop_identity")?, - declaring_module: record_field_string(&ctx, fields.as_slice(), "declaring_module")?, - service: record_field_string(&ctx, fields.as_slice(), "service")?, - operation_qualified: record_field_string(&ctx, fields.as_slice(), "operation_qualified")?, - operation_bare: record_field_string(&ctx, fields.as_slice(), "operation_bare")?, - }) -} - -fn capture_gap_exclusion() -> Result<&'static CaptureGapExclusion, &'static str> { - static GAP: OnceLock> = OnceLock::new(); - match GAP.get_or_init(load_capture_gap_exclusion) { - Ok(gap) => Ok(gap), - Err(e) => Err(e.as_str()), - } -} - -/// Facts `v1.compiler.emit` `shell_emission_refusal_fact` renders for -/// `ShellChannelNotRealizedByTarget` on the three stderr-accounting channels the drop names. -/// Not every channel `shell_channel_realized_by_target` currently returns false for: a later -/// unrealized stdout-side channel must still refuse the union (review 77034). -fn rust_stderr_capture_channel_not_realized_facts() -> &'static BTreeSet { - static FACTS: OnceLock> = OnceLock::new(); - FACTS.get_or_init(|| { - use crate::v1_compiler_artifact::RenderTarget; - use crate::v1_compiler_emit::{ - render_target_name, shell_channel_realized_by_target, shell_emission_refusal_fact, - shell_result_channel_key, ShellEmissionRefusal, ShellResultChannel, - }; - let target = RenderTarget::Rust; - let target_name = render_target_name(target); - [ - ShellResultChannel::ShellChanStderrTruncated, - ShellResultChannel::ShellChanStderrTotalBytes, - ShellResultChannel::ShellChanStderrRetainedBytes, - ] - .into_iter() - .filter(|channel| !shell_channel_realized_by_target(*channel, target)) - .map(|channel| { - shell_emission_refusal_fact(Rc::new( - ShellEmissionRefusal::ShellChannelNotRealizedByTarget { - key: shell_result_channel_key(channel), - target_name: target_name.clone(), - }, - )) - }) - .collect() - }) -} - -/// The drop population: rust shell TransportEmissionNotModeled on exactly -/// `extdeps.gunbc` `WitnessBin.Run` whose fact equals a ShellChannelNotRealizedByTarget -/// fact for an unrealized rust channel. Any other module, service, or operation stays rendered. -fn stderr_capture_policy_gap_service(d: &Rc) -> Option<(String, String)> { - let Ok(gap) = capture_gap_exclusion() else { - return None; - }; - match &*d.diagnostic { - crate::v1_std_core::CompilerDiagnostic::TransportEmissionNotModeled { - transport_kind, - service, - operation, - declaring_module, - target, - missing_realization_fact, - .. - } if transport_kind == "shell" - && target == "rust" - && declaring_module.as_str() == gap.declaring_module - && service.as_str() == gap.service - && is_drop_run_operation(operation) - && rust_stderr_capture_channel_not_realized_facts() - .contains(missing_realization_fact) => - { - Some((declaring_module.clone(), service.clone())) - } - _ => None, - } -} - -fn is_drop_run_operation(operation: &str) -> bool { - let Ok(gap) = capture_gap_exclusion() else { - return false; - }; - operation == gap.operation_bare || operation == gap.operation_qualified -} - -fn transport_emission_run(d: &Rc) -> Option<(String, String)> { - match &*d.diagnostic { - crate::v1_std_core::CompilerDiagnostic::TransportEmissionNotModeled { - service, - operation, - declaring_module, - .. - } if is_drop_run_operation(operation) => Some((declaring_module.clone(), service.clone())), - _ => None, - } -} - -/// The selection fold: among supplied unmodeled-transport rows, keep `(module, service)` -/// only when every Run row for that pair is a capture-gap fact. Production feeds it the -/// three emit diagnostic streams; tests supply rows. -fn select_run_operations_excluded_for_stderr_capture_gap( - unmodeled: &[Rc], -) -> BTreeSet<(String, String)> { - let mut gap_runs: BTreeSet<(String, String)> = BTreeSet::new(); - for d in unmodeled { - if let Some(key) = stderr_capture_policy_gap_service(d) { - gap_runs.insert(key); - } - } - gap_runs - .into_iter() - .filter(|key| { - unmodeled - .iter() - .filter(|d| transport_emission_run(d).as_ref() == Some(key)) - .all(|d| stderr_capture_policy_gap_service(d).is_some()) - }) - .collect() -} - -/// `extdeps.gunbc` `gunbc.WitnessBin.Run` when that operation's unmodeled-transport refusals -/// are solely the rust stderr-capture gap. Other operations on the same service are not members. -fn run_operations_excluded_for_stderr_capture_gap( - typed: &Rc, -) -> BTreeSet<(String, String)> { - let target = crate::v1_compiler_artifact::RenderTarget::Rust; - let mut unmodeled = Vec::new(); - unmodeled.extend( - crate::v1_compiler_emit::unmodeled_file_transport_diagnostics(typed.clone(), target) - .iter() - .cloned(), - ); - unmodeled.extend( - crate::v1_compiler_emit::unmodeled_shell_transport_diagnostics(typed.clone(), target) - .iter() - .cloned(), - ); - unmodeled.extend( - crate::v1_compiler_emit::unmodeled_rest_transport_diagnostics(typed.clone(), target) - .iter() - .cloned(), - ); - select_run_operations_excluded_for_stderr_capture_gap(&unmodeled) -} - -fn strip_excluded_run_operations( - typed: Rc, - excluded: &BTreeSet<(String, String)>, -) -> Rc { - if excluded.is_empty() { - return typed; - } - let modules = Rc::new( - typed - .modules - .iter() - .map(|tm| { - let module_name = crate::v1_compiler_infer_env::authored_name( - tm.type_env.clone(), - tm.module.clone(), - ); - let items = Rc::new( - tm.items - .iter() - .filter_map(|item| { - if item.module_item_kind - != crate::v1_std_core::ParsedModuleItemKind::ModuleItemService - { - return Some((*item).clone()); - } - let service = crate::v1_compiler_infer_env::authored_name( - tm.type_env.clone(), - (*item).clone(), - ); - if !excluded.contains(&(module_name.clone(), service)) { - return Some((*item).clone()); - } - let kept: im::Vector<_> = item - .children - .iter() - .filter(|op| { - !is_drop_run_operation( - &crate::v1_compiler_infer_env::authored_name( - tm.type_env.clone(), - (*op).clone(), - ), - ) - }) - .cloned() - .collect(); - if kept.is_empty() { - return None; - } - Some(Rc::new({ - let mut node = (**item).clone(); - node.children = Rc::new(kept); - node - })) - }) - .collect::>(), - ); - Rc::new(crate::v1_compiler_infer_items::TypedModule { - items, - ..(**tm).clone() - }) - }) - .collect::>(), - ); - Rc::new(crate::v1_compiler_infer_items::ResolvedGraph { - modules, - ..(*typed).clone() - }) -} - -/// Compile stays the full closure. Emit strips only `gunbc.WitnessBin.Run` when its rust -/// refusals are solely the capture-policy gap. Other operations on that service stay. -fn union_emit_graph_excluding_unmodeled_stderr_capture( - resolved: Rc, -) -> (Rc, Vec) { - let Some(typed) = resolved.graph.clone() else { - return (resolved, Vec::new()); - }; - let excluded_keys = run_operations_excluded_for_stderr_capture_gap(&typed); - if excluded_keys.is_empty() { - return (resolved, Vec::new()); - } - let Ok(gap) = capture_gap_exclusion() else { - return (resolved, Vec::new()); - }; - let excluded: Vec = excluded_keys - .iter() - .map(|(module, service)| { - format!( - "module={module} service={service} operation={} \ - cause=ShellChannelNotRealizedByTarget \ - fact=stderr_capture_policy_unrealized drop={}", - gap.operation_qualified, gap.drop_identity, - ) - }) - .collect(); - let graph = strip_excluded_run_operations(typed, &excluded_keys); - ( - Rc::new(v1_compiler_compile::ResolvedPipelineResult { - graph: Some(graph), - ..(*resolved).clone() - }), - excluded, - ) -} - /// The red control's member: a non-tail effectful self-call, which the rust emitter refuses /// (`EffectfulSelfRecursionUnrealized`, the derived form exercised by /// `test.claim.effectful_item_kind_collapse_witness_test`). @@ -3708,8 +3347,8 @@ pub(crate) fn fixture_closure_union_controls() -> Result<(u128, u128), String> { ))) } } - // A fixture whose closure reaches extdeps.gunbc is admitted with the typed capture-policy - // exclusion, not as FixtureClosureUnionEmitRefused (the #13420 floor after #13437). + // A fixture whose closure reaches extdeps.gunbc is admitted by rust emit: WitnessBin.Run + // is realized, and the retired capture-gap strip is gone (review 77726). let gunbc_observed = fixture_closure_union_emit_receipt( &fixture_closure_union_control_union(FIXTURE_CLOSURE_GUNBC_REACH_MEMBER) .map_err(&refuse)?, @@ -3719,21 +3358,13 @@ pub(crate) fn fixture_closure_union_controls() -> Result<(u128, u128), String> { "a fixture whose closure reaches extdeps.gunbc refused: {refusal}" )) })?; - if !gunbc_observed.excluded.iter().any(|row| { - let Ok(gap) = capture_gap_exclusion() else { - return false; - }; - row.contains(&format!("module={}", gap.declaring_module)) - && row.contains(&format!("operation={}", gap.operation_qualified)) - && row.contains("cause=ShellChannelNotRealizedByTarget") - && row.contains(&gap.drop_identity) - }) { + if !gunbc_observed.excluded.is_empty() { return Err(refuse(format!( - "a fixture whose closure reaches extdeps.gunbc was admitted without the typed exclusion: {gunbc_observed:?}" + "the retired capture-gap strip must not still exclude a member: {gunbc_observed:?}" ))); } // A real emit error in a member the union still renders still refuses, even when the same - // closure also reaches the excluded service. + // closure also reaches extdeps.gunbc. match fixture_closure_union_emit_receipt( &fixture_closure_union_control_union(FIXTURE_CLOSURE_GUNBC_AND_REAL_EMIT_ERROR) .map_err(&refuse)?, @@ -3748,7 +3379,7 @@ pub(crate) fn fixture_closure_union_controls() -> Result<(u128, u128), String> { } Ok(observed) => { return Err(refuse(format!( - "a real emit error beside the excluded gunbc service did not refuse the union: {observed:?}" + "a real emit error beside a gunbc-reaching closure did not refuse the union: {observed:?}" ))) } } @@ -3778,10 +3409,6 @@ mod fixture_closure_union_tests { /// The recorder and the union are process-wide; tests that touch them run one at a time. static UNION_TEST_LOCK: Mutex<()> = Mutex::new(()); - fn gap_excl() -> &'static CaptureGapExclusion { - capture_gap_exclusion().unwrap_or_else(|e| panic!("{e}")) - } - /// An empty union is a bypassed recording seam and refuses (review 76399). #[test] fn an_empty_union_refuses() { @@ -3886,340 +3513,362 @@ mod fixture_closure_union_tests { assert!(union.conflicts.contains("dag/a.dag")); } + const STDERR_CAPTURE_MEMBER: &str = "module efr_member\nimport std.shell_stream_capture { WitnessStderrCapturePolicy, BoundedTail }\nimport std.measure { byte_size }\nservice Bin {\n operation Run {\n input {\n bin_path: String\n stderr_capture: WitnessStderrCapturePolicy = BoundedTail { bytes: byte_size(count: 16) }\n }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n stderr_total_bytes: Int from \"stderr_total_bytes\"\n stderr_retained_bytes: Int from \"stderr_retained_bytes\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const STDERR_CAPTURE_UNMODELED_SIBLING: &str = "module efr_member\nimport std.shell_stream_capture { WitnessStderrCapturePolicy, BoundedTail }\nimport std.measure { byte_size }\nservice Bin {\n operation Run {\n input {\n bin_path: String\n stderr_capture: WitnessStderrCapturePolicy = BoundedTail { bytes: byte_size(count: 16) }\n }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n stderr_total_bytes: Int from \"stderr_total_bytes\"\n stderr_retained_bytes: Int from \"stderr_retained_bytes\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n operation Weird {\n input { bin_path: String }\n output { digest: String from \"stderr_digest_hex\" }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const STDERR_CAPTURE_WITHOUT_POLICY: &str = "module efr_member\nservice Bin {\n operation Run {\n input { bin_path: String }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const STDERR_CAPTURE_STRING_POLICY: &str = "module efr_member\nservice Bin {\n operation Run {\n input {\n bin_path: String\n stderr_capture: String\n }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const STDERR_CAPTURE_OPTIONAL_POLICY: &str = "module efr_member\nimport std.shell_stream_capture { WitnessStderrCapturePolicy }\nservice Bin {\n operation Run {\n input {\n bin_path: String\n stderr_capture: WitnessStderrCapturePolicy?\n }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const STDERR_CAPTURE_COLLECTION_POLICY: &str = "module efr_member\nimport std.shell_stream_capture { WitnessStderrCapturePolicy }\nservice Bin {\n operation Run {\n input {\n bin_path: String\n stderr_capture: List\n }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const STDERR_CAPTURE_HOMONYM_POLICY: &str = "module efr_member\ntype WitnessStderrCapturePolicy { mark: String }\nservice Bin {\n operation Run {\n input {\n bin_path: String\n stderr_capture: WitnessStderrCapturePolicy\n }\n output {\n success: Bool from \"exit_success\"\n stderr_truncated: Bool from \"stderr_truncated\"\n }\n transport shell { argv: [\"{bin_path}\"] }\n }\n}\n"; + + const GUNBC_MODULE_REACH_MEMBER: &str = + "module efr_member\nimport extdeps.gunbc { packages }\nfn ignore() -> Int { 0 }\n"; + #[test] - fn capture_gap_keys_on_shell_channel_not_realized_fact_equality() { - use crate::v1_compiler_emit::{shell_emission_refusal_fact, ShellEmissionRefusal}; - use crate::v1_std_core::{make_error_node, CompilerDiagnostic}; - let span = crate::v1_std_core::kernel_span("probe".to_string()); - let mk = |fact: String| { - let gap = gap_excl(); - make_error_node( - Rc::new(CompilerDiagnostic::TransportEmissionNotModeled { - transport_kind: "shell".to_string(), - service: gap.service.to_string(), - operation: gap.operation_qualified.to_string(), - declaring_module: gap.declaring_module.to_string(), - target: "rust".to_string(), - missing_realization_fact: fact, - span: span.clone(), - }), - gap.declaring_module.to_string(), - ) - }; - let gap = shell_emission_refusal_fact(Rc::new( - ShellEmissionRefusal::ShellChannelNotRealizedByTarget { - key: "stderr_truncated".to_string(), - target_name: "rust".to_string(), - }, - )); - let unmodeled_key = - shell_emission_refusal_fact(Rc::new(ShellEmissionRefusal::ShellOutputKeyNotModeled { - key: "not_a_channel".to_string(), - })); - let substring_poison = - "unmodeled key 'not_a_channel' implements no stderr capture policy".to_string(); - let stdout_unrealized = shell_emission_refusal_fact(Rc::new( - ShellEmissionRefusal::ShellChannelNotRealizedByTarget { - key: "stdout".to_string(), - target_name: "rust".to_string(), - }, - )); - assert!(stderr_capture_policy_gap_service(&mk(gap.clone())).is_some()); - assert!(stderr_capture_policy_gap_service(&mk(unmodeled_key)).is_none()); - assert!(stderr_capture_policy_gap_service(&mk(substring_poison)).is_none()); - assert!(stderr_capture_policy_gap_service(&mk(stdout_unrealized)).is_none()); - let other_module = make_error_node( - Rc::new(CompilerDiagnostic::TransportEmissionNotModeled { - transport_kind: "shell".to_string(), - service: gap_excl().service.to_string(), - operation: gap_excl().operation_qualified.to_string(), - declaring_module: "extdeps.other".to_string(), - target: "rust".to_string(), - missing_realization_fact: gap, - span: span.clone(), - }), - "extdeps.other".to_string(), + fn declared_stderr_capture_channels_do_not_refuse_the_union() { + let union = fixture_closure_union_control_union(STDERR_CAPTURE_MEMBER) + .expect("capture member resolves"); + fixture_closure_union_emit_receipt(&union) + .expect("the rust shell handler realizes the declared capture channels"); + } + + #[test] + fn an_unmodeled_shell_channel_still_refuses_the_union() { + let union = fixture_closure_union_control_union(STDERR_CAPTURE_UNMODELED_SIBLING) + .expect("mixed member resolves"); + let refusal = fixture_closure_union_emit_receipt(&union) + .expect_err("unmodeled stderr_digest_hex must still refuse"); + assert!( + refusal.contains("cause=FixtureClosureUnionEmitRefused") + && refusal.contains("stderr_digest_hex"), + "{refusal}" ); - assert!(stderr_capture_policy_gap_service(&other_module).is_none()); } - fn gap_fact() -> String { - use crate::v1_compiler_emit::{shell_emission_refusal_fact, ShellEmissionRefusal}; - shell_emission_refusal_fact(Rc::new( - ShellEmissionRefusal::ShellChannelNotRealizedByTarget { - key: "stderr_truncated".to_string(), - target_name: "rust".to_string(), - }, - )) + #[test] + fn capture_channels_without_stderr_capture_input_refuse_the_union() { + let union = fixture_closure_union_control_union(STDERR_CAPTURE_WITHOUT_POLICY) + .expect("member without policy input resolves"); + let refusal = fixture_closure_union_emit_receipt(&union) + .expect_err("a capture channel without stderr_capture must refuse"); + assert!( + refusal.contains("cause=FixtureClosureUnionEmitRefused") + && refusal.contains("stderr_capture") + && refusal.contains("transport emission is not modeled"), + "{refusal}" + ); } - fn transport_row( - module: &str, - service: &str, - operation: &str, - fact: String, - ) -> Rc { - crate::v1_std_core::make_error_node( - Rc::new( - crate::v1_std_core::CompilerDiagnostic::TransportEmissionNotModeled { - transport_kind: "shell".to_string(), - service: service.to_string(), - operation: operation.to_string(), - declaring_module: module.to_string(), - target: "rust".to_string(), - missing_realization_fact: fact, - span: crate::v1_std_core::kernel_span("probe".to_string()), - }, - ), - module.to_string(), - ) + fn capture_channels_refuse_unless_stderr_capture_is_the_required_scalar_policy( + source: &str, + label: &str, + ) { + let union = fixture_closure_union_control_union(source).unwrap_or_else(|e| { + panic!("{label} must resolve so emit can refuse the typed policy: {e}") + }); + let refusal = fixture_closure_union_emit_receipt(&union) + .expect_err("wrong stderr_capture shape must refuse at emit, not rustc"); + assert!( + refusal.contains("cause=FixtureClosureUnionEmitRefused") + && refusal.contains("transport emission is not modeled") + && refusal.contains("stderr_capture"), + "{label}: {refusal}" + ); + assert!( + !refusal.contains("error[E") && !refusal.contains("mismatched types"), + "typed emission must refuse before rustc: {label}: {refusal}" + ); } #[test] - fn selection_fold_allows_only_run_capture_rows_and_vetoes_mixed_refusal() { - let gap = gap_fact(); - let gap_row = gap_excl(); - let allowed = transport_row( - &gap_row.declaring_module, - &gap_row.service, - &gap_row.operation_qualified, - gap.clone(), - ); - let selected = select_run_operations_excluded_for_stderr_capture_gap(&[allowed]); - assert_eq!( - selected.iter().cloned().collect::>(), - vec![( - gap_row.declaring_module.to_string(), - gap_row.service.to_string() - )] + fn capture_channels_with_string_stderr_capture_refuse_the_union() { + capture_channels_refuse_unless_stderr_capture_is_the_required_scalar_policy( + STDERR_CAPTURE_STRING_POLICY, + "String", ); + } - let mixed_key = crate::v1_compiler_emit::shell_emission_refusal_fact(Rc::new( - crate::v1_compiler_emit::ShellEmissionRefusal::ShellOutputKeyNotModeled { - key: "not_a_channel".to_string(), - }, - )); - let mixed = vec![ - transport_row( - &gap_row.declaring_module, - &gap_row.service, - &gap_row.operation_qualified, - gap.clone(), - ), - transport_row( - &gap_row.declaring_module, - &gap_row.service, - &gap_row.operation_qualified, - mixed_key, - ), - ]; - assert!(select_run_operations_excluded_for_stderr_capture_gap(&mixed).is_empty()); - - let wrong_module = transport_row( - "extdeps.other", - &gap_row.service, - &gap_row.operation_qualified, - gap.clone(), + #[test] + fn capture_channels_with_optional_stderr_capture_refuse_the_union() { + capture_channels_refuse_unless_stderr_capture_is_the_required_scalar_policy( + STDERR_CAPTURE_OPTIONAL_POLICY, + "optional", ); - assert!(select_run_operations_excluded_for_stderr_capture_gap(&[wrong_module]).is_empty()); + } - let nonmember = transport_row(&gap_row.declaring_module, &gap_row.service, "Sibling", gap); - assert!(select_run_operations_excluded_for_stderr_capture_gap(&[nonmember]).is_empty()); + #[test] + fn capture_channels_with_collection_stderr_capture_refuse_the_union() { + capture_channels_refuse_unless_stderr_capture_is_the_required_scalar_policy( + STDERR_CAPTURE_COLLECTION_POLICY, + "collection", + ); } - fn kernel_named( - name: &str, - kind: crate::v1_std_core::ParsedModuleItemKind, - children: im::Vector>, - ) -> Rc { - let mut node = (*crate::v1_std_core::leaf_node_with_span( - Rc::new(crate::std_occurrence_identity::NodeOccurrenceIdentity::OccurrenceSynthetic), - name.to_string(), - crate::v1_std_core::kernel_span(name.to_string()), - )) - .clone(); - node.module_item_kind = kind; - node.children = Rc::new(children); - Rc::new(node) + #[test] + fn capture_channels_with_homonym_stderr_capture_refuse_the_union() { + capture_channels_refuse_unless_stderr_capture_is_the_required_scalar_policy( + STDERR_CAPTURE_HOMONYM_POLICY, + "homonym", + ); } - fn supplied_graph_with_run_and_sibling() -> Rc { - use crate::v1_compiler_infer_items::{ - ModuleInterface, ModuleTypecheckProgress, ResolvedGraph, TypedModule, + fn rustc_and_run_emitted_capture( + stem: &str, + complete_limit: Option, + tail_bytes: usize, + stderr_payload: &str, + stdin_payload: Option<&str>, + ) -> std::process::Output { + let limit = match complete_limit { + Some(n) => format!("Some({n})"), + None => "None".to_string(), + }; + let (script, stdin_prelude) = match stdin_payload { + Some(stdin) => { + let n = stderr_payload.len(); + let combined = format!("{}{}", stderr_payload, stdin); + ( + format!("head -c {n} 1>&2; cat >/dev/null"), + format!( + "let __stdin_bytes: Vec = ({:?}).as_bytes().to_vec();\n\ + let mut stdin_pipe = output.stdin.take();\n\ + let __stdin_thread = std::thread::spawn(move || -> std::io::Result<()> {{\n\ + use std::io::Write;\n\ + if let Some(mut stdin) = stdin_pipe {{\n\ + stdin.write_all(&__stdin_bytes)?;\n\ + }}\n\ + Ok(())\n\ + }});\n", + combined + ), + ) + } + None => ( + "printf %s '{payload}' 1>&2".to_string(), + "let mut stdin_pipe = output.stdin.take();\n\ + let __stdin_thread = std::thread::spawn(move || -> std::io::Result<()> {\n\ + drop(stdin_pipe);\n\ + Ok(())\n\ + });\n" + .to_string(), + ), }; - use crate::v1_std_core::ParsedModuleItemKind; - let env = crate::v1_compiler_infer_env::empty_type_env(); - let cache = crate::v1_compiler_infer_env::empty_type_env_cache(); - let gap = gap_excl(); - let run = kernel_named( - &gap.operation_bare, - ParsedModuleItemKind::NotAModuleItem, - im::vector![], + let drain = format!( + "{}{}{}", + crate::v1_compiler_emit_rust::shell_capture_drain_start(), + stdin_prelude, + crate::v1_compiler_emit_rust::shell_capture_join_project() ); - let sibling = kernel_named( - "Sibling", - ParsedModuleItemKind::NotAModuleItem, - im::vector![], + let program = format!( + "fn main() -> Result<(), Box> {{\n\ + let __stderr_complete_limit: Option = {limit};\n\ + let __stderr_complete_source: String = {source};\n\ + let __stderr_tail_bytes: usize = {tail};\n\ + let mut output = std::process::Command::new(\"sh\")\n\ + .args([\"-c\", \"{script}\"])\n\ + .stdin(std::process::Stdio::piped())\n\ + .stdout(std::process::Stdio::piped())\n\ + .stderr(std::process::Stdio::piped())\n\ + .spawn()?;\n\ + {drain}\n\ + print!(\"{{stderr_truncated}}|{{stderr_total_bytes}}|{{stderr_retained_bytes}}|{{stderr}}\");\n\ + Ok(())\n\ + }}\n", + script = script.replace("{payload}", stderr_payload), + limit = limit, + source = if complete_limit.is_some() { + "\"host budget\".to_string()".to_string() + } else { + "String::new()".to_string() + }, + tail = tail_bytes, + drain = drain, ); - let other_fn = kernel_named( - "unrelated_fn", - ParsedModuleItemKind::ModuleItemFunction, - im::vector![], + let root = std::env::temp_dir().join(format!( + "gunbc-emitted-capture-{}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(), + stem + )); + std::fs::create_dir_all(&root).expect("scratch dir"); + let src = root.join("main.rs"); + let exe = root.join("specimen"); + std::fs::write(&src, program).expect("write emitted capture harness"); + let compiled = std::process::Command::new("rustc") + .args(["--edition=2021", "-o"]) + .arg(&exe) + .arg(&src) + .output() + .expect("invoke rustc"); + assert!( + compiled.status.success(), + "emitted capture body refused to compile: {}", + String::from_utf8_lossy(&compiled.stderr) ); - let service = kernel_named( - &gap.service, - ParsedModuleItemKind::ModuleItemService, - im::vector![run, sibling], + let run = std::process::Command::new(&exe) + .output() + .expect("run emitted capture specimen"); + let _ = std::fs::remove_dir_all(&root); + run + } + + #[test] + fn emitted_bounded_tail_truncates_and_keeps_the_tail() { + let run = rustc_and_run_emitted_capture( + "over-tail", + None, + 16, + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + None, ); - let module = kernel_named( - &gap.declaring_module, - ParsedModuleItemKind::NotAModuleItem, - im::vector![], + assert!( + run.status.success(), + "over-tail specimen failed: {}", + String::from_utf8_lossy(&run.stderr) ); - let other_module_node = kernel_named( - "other.mod", - ParsedModuleItemKind::NotAModuleItem, - im::vector![], + assert_eq!( + String::from_utf8_lossy(&run.stdout), + "true|36|16|UVWXYZ0123456789" ); - let other_item = kernel_named( - "KeepMe", - ParsedModuleItemKind::ModuleItemFunction, - im::vector![], + } + + #[test] + fn emitted_bounded_tail_under_cap_is_complete() { + let run = rustc_and_run_emitted_capture("under-cap", None, 16, "abcdefghij", None); + assert!( + run.status.success(), + "under-cap specimen failed: {}", + String::from_utf8_lossy(&run.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&run.stdout), + "false|10|10|abcdefghij" ); - let interface = |e: Rc, - c: Rc, - path: &str| { - Rc::new(ModuleInterface { - summary: Rc::new(crate::std_interface_summary::InterfaceSummary { - module_path: path.to_string(), - exports: Rc::new(im::vector![]), - interface_hash: crate::std_interface_summary::interface_summary_rollup( - Rc::new(im::vector![]), - ), - }), - env: e, - cache: c, - }) - }; - let tm = Rc::new(TypedModule { - module, - items: Rc::new(im::vector![service, other_fn]), - progress: ModuleTypecheckProgress::ItemsChecked, - type_env: env.clone(), - type_env_cache: cache.clone(), - interface: interface(env.clone(), cache.clone(), &gap.declaring_module), - func_env: Rc::new(crate::v1_compiler_infer_sigs::ResolvedFuncEnv { - name: gap.declaring_module.to_string(), - local: crate::v1_rt::rc_empty_map(), - parents: Rc::new(im::vector![]), - }), - item_registry: crate::v1_rt::rc_empty_map(), - occurrence_transport: None, - }); - let other = Rc::new(TypedModule { - module: other_module_node, - items: Rc::new(im::vector![other_item]), - progress: ModuleTypecheckProgress::ItemsChecked, - type_env: env.clone(), - type_env_cache: cache.clone(), - interface: interface(env, cache, "other.mod"), - func_env: Rc::new(crate::v1_compiler_infer_sigs::ResolvedFuncEnv { - name: "other.mod".to_string(), - local: crate::v1_rt::rc_empty_map(), - parents: Rc::new(im::vector![]), - }), - item_registry: crate::v1_rt::rc_empty_map(), - occurrence_transport: None, - }); - Rc::new(ResolvedGraph { - modules: Rc::new(im::vector![tm, other]), - item_registry: crate::v1_rt::rc_empty_map(), - item_leaf_owner_modules: crate::v1_rt::rc_empty_map(), - diagnostics: Rc::new(im::vector![]), - }) } - fn service_op_names( - graph: &crate::v1_compiler_infer_items::ResolvedGraph, - module: &str, - service: &str, - ) -> Vec { - let env = crate::v1_compiler_infer_env::empty_type_env(); - graph - .modules - .iter() - .find(|tm| { - crate::v1_compiler_infer_env::authored_name(tm.type_env.clone(), tm.module.clone()) - == module - }) - .into_iter() - .flat_map(|tm| tm.items.iter()) - .filter(|item| { - item.module_item_kind == crate::v1_std_core::ParsedModuleItemKind::ModuleItemService - && crate::v1_compiler_infer_env::authored_name(env.clone(), (*item).clone()) - == service - }) - .flat_map(|item| { - item.children - .iter() - .map(|op| crate::v1_compiler_infer_env::authored_name(env.clone(), op.clone())) - }) - .collect() + #[test] + fn emitted_complete_over_budget_refuses() { + let run = + rustc_and_run_emitted_capture("complete-over", Some(8), 0, "abcdefghijklmnop", None); + assert!( + !run.status.success(), + "Complete overflow must refuse, got stdout {:?}", + String::from_utf8_lossy(&run.stdout) + ); + let err = String::from_utf8_lossy(&run.stderr); + assert!( + err.contains("WitnessStderrCaptureCompleteBudgetExceeded") + && err.contains("16") + && err.contains("8"), + "{err}" + ); } #[test] - fn strip_removes_only_run_and_keeps_sibling_and_unrelated_items() { - let gap = gap_excl(); - let graph = supplied_graph_with_run_and_sibling(); - let mut excluded = BTreeSet::new(); - excluded.insert((gap.declaring_module.to_string(), gap.service.to_string())); - let stripped = strip_excluded_run_operations(graph.clone(), &excluded); + fn emitted_complete_under_budget_retains_all() { + let run = rustc_and_run_emitted_capture("complete-under", Some(64), 0, "abcdefghij", None); + assert!( + run.status.success(), + "under-budget Complete failed: {}", + String::from_utf8_lossy(&run.stderr) + ); assert_eq!( - service_op_names(&stripped, &gap.declaring_module, &gap.service), - vec!["Sibling".to_string()] + String::from_utf8_lossy(&run.stdout), + "false|10|10|abcdefghij" ); - let env = crate::v1_compiler_infer_env::empty_type_env(); - let gunbc_item_names: Vec = stripped - .modules - .iter() - .find(|tm| { - crate::v1_compiler_infer_env::authored_name(tm.type_env.clone(), tm.module.clone()) - == gap.declaring_module - }) - .unwrap() - .items - .iter() - .map(|item| crate::v1_compiler_infer_env::authored_name(env.clone(), item.clone())) - .collect(); + } + + #[test] + fn emitted_stdin_and_long_stderr_does_not_deadlock() { + let stderr = "Y".repeat(80_000); + let stdin = "X".repeat(80_000); + let run = + rustc_and_run_emitted_capture("stdin-long-stderr", None, 16, &stderr, Some(&stdin)); assert!( - gunbc_item_names.contains(&"unrelated_fn".to_string()), - "{gunbc_item_names:?}" + run.status.success(), + "stdin+stderr specimen deadlocked or failed: {}", + String::from_utf8_lossy(&run.stderr) ); - assert_eq!(stripped.modules.len(), 2, "unrelated module must remain"); - let empty = strip_excluded_run_operations(graph, &BTreeSet::new()); assert_eq!( - service_op_names(&empty, &gap.declaring_module, &gap.service), - vec![gap.operation_bare.to_string(), "Sibling".to_string()] + String::from_utf8_lossy(&run.stdout), + format!("true|80000|16|{}", "Y".repeat(16)) ); } #[test] - fn capture_gap_exclusion_reads_typed_fields_from_the_drop_module() { - let gap = gap_excl(); - assert!(!gap.drop_identity.is_empty()); - assert!(!gap.declaring_module.is_empty()); - assert!(!gap.service.is_empty()); - assert!(!gap.operation_bare.is_empty()); + fn emitted_absent_policy_refuses_before_spawn() { + let refusal = crate::v1_compiler_emit::shell_emission_refusal_fact(std::rc::Rc::new( + crate::v1_compiler_emit::ShellEmissionRefusal::ShellCapturePolicyInputAbsent { + key: "stderr_truncated".to_string(), + }, + )); + let marker = std::env::temp_dir().join(format!( + "gunbc-absent-policy-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let program = format!( + "fn main() -> Result<(), Box> {{\n\ + return Err(\"{refusal}\".into());\n\ + let _ = std::process::Command::new(\"sh\")\n\ + .args([\"-c\", \"printf ran > {marker}\"])\n\ + .status()?;\n\ + Ok(())\n\ + }}\n", + refusal = refusal.replace('\\', "\\\\").replace('"', "\\\""), + marker = marker.display(), + ); + let root = + std::env::temp_dir().join(format!("gunbc-absent-policy-src-{}", std::process::id())); + std::fs::create_dir_all(&root).expect("scratch"); + let src = root.join("main.rs"); + let exe = root.join("specimen"); + std::fs::write(&src, program).expect("write"); + let compiled = std::process::Command::new("rustc") + .args(["--edition=2021", "-o"]) + .arg(&exe) + .arg(&src) + .output() + .expect("rustc"); + assert!( + compiled.status.success(), + "{}", + String::from_utf8_lossy(&compiled.stderr) + ); + let run = std::process::Command::new(&exe).output().expect("run"); + let _ = std::fs::remove_dir_all(&root); + assert!(!run.status.success(), "absent policy must refuse"); + assert!( + String::from_utf8_lossy(&run.stderr).contains("stderr_capture"), + "{}", + String::from_utf8_lossy(&run.stderr) + ); assert!( - gap.operation_qualified == gap.operation_bare - || gap - .operation_qualified - .ends_with(&format!(".{}", gap.operation_bare)), - "qualified={} bare={}", - gap.operation_qualified, - gap.operation_bare + !marker.exists(), + "the child command ran; policy must refuse before spawn" ); } + + #[test] + fn reaching_extdeps_gunbc_does_not_refuse_the_union_for_capture_channels() { + let union = fixture_closure_union_control_union(GUNBC_MODULE_REACH_MEMBER) + .expect("extdeps.gunbc reach resolves"); + fixture_closure_union_emit_receipt(&union).unwrap_or_else(|refusal| { + panic!("a compile-probe reach of extdeps.gunbc must emit: {refusal}"); + }); + } } #[cfg(test)] diff --git a/src/v1/stage0/src/memory_governor.rs b/src/v1/stage0/src/memory_governor.rs index 3f38f189c31..bc9eadf2827 100644 --- a/src/v1/stage0/src/memory_governor.rs +++ b/src/v1/stage0/src/memory_governor.rs @@ -1218,17 +1218,10 @@ pub fn host_budget_unreadable_reason() -> String { ) } -/// Resolve the host budget from OBSERVATIONS, so every arm — including the refusal — is -/// reachable from a test on any machine. `read_host_budget_resolution` is this function -/// applied to the real reads; nothing else composes the precedence. -/// -/// The effective planning ceiling is the minimum of the operator request and every observed -/// applicable cgroup line. An operator request alone is `DeclaredUnverified`: an integer in an -/// environment variable constrains no allocation and is not evidence of executor provisioning. -/// There is no meminfo arm: MemAvailable and MemTotal describe a MACHINE, and -/// on a kernel that can express a private limit, substituting one for the limit this process -/// failed to read is DESIGN §5's absorbing fallback (answering with a superset). Authority: -/// `gunbc.host_budget_source` `host_budget_source_admissible_as_bound_on_kernel`. +/// Typed view of `v1_rt::resolve_host_budget_join`. That function is the one composer +/// (`gunbc.host_budget_source`); this maps its result onto `HostBudgetResolution` so +/// existing governor consumers keep a typed source. Tests plant observations here +/// and still exercise the runtime join. pub fn resolve_host_budget( env_override: Option, cgroup_high: Option<(String, u64)>, @@ -1236,99 +1229,94 @@ pub fn resolve_host_budget( cgroup_v1_limit: Option<(String, CgroupV1MemoryLimitValue)>, darwin_physical: Option, ) -> HostBudgetResolution { - // A v1 memory hierarchy holds this process but its limit could not be read: that limit may - // be the tightest one, so no other observation may stand in for it (DESIGN §5). - let cgroup_v1_limit = match cgroup_v1_limit { - Some((dir, CgroupV1MemoryLimitValue::Unparseable(body))) => { - return HostBudgetResolution::Unreadable { - reason: format!( - "cgroup v1 memory hierarchy at {dir} holds this process but its \ - hierarchical_memory_limit is unreadable ({body}); a bound that may be the \ - tightest cannot be replaced by another reading" - ), - }; - } - Some((dir, CgroupV1MemoryLimitValue::Limited(bytes))) => Some((dir, bytes)), - Some((_, CgroupV1MemoryLimitValue::Unlimited)) | None => None, - }; - // Every observed process-scoped line is a candidate; the tightest one is the planning - // ceiling. On a hybrid host the unified hierarchy carries no memory files, so the v2 and - // v1 readings do not normally coexist; when they do, the minimum is still the honest bound. - let observation = [ - cgroup_high.map(|(cgroup_dir, b)| (HostBudgetSource::CgroupMemoryHigh { cgroup_dir }, b)), - cgroup_max.map(|(cgroup_dir, b)| (HostBudgetSource::CgroupMemoryMax { cgroup_dir }, b)), - cgroup_v1_limit.map(|(cgroup_dir, b)| { - ( - HostBudgetSource::CgroupV1HierarchicalMemoryLimit { cgroup_dir }, - b, - ) - }), - ] - .into_iter() - .flatten() - .fold(None::<(HostBudgetSource, u64)>, |best, cand| match best { - Some(cur) if cur.1 <= cand.1 => Some(cur), - _ => Some(cand), + let v1 = cgroup_v1_limit.map(|(dir, v)| { + ( + dir, + match v { + CgroupV1MemoryLimitValue::Limited(bytes) => { + crate::v1_rt::HostBudgetCgroupV1::Limited(bytes) + } + CgroupV1MemoryLimitValue::Unlimited => crate::v1_rt::HostBudgetCgroupV1::Unlimited, + CgroupV1MemoryLimitValue::Unparseable(body) => { + crate::v1_rt::HostBudgetCgroupV1::Unparseable(body) + } + }, + ) }); - if let Some((source, observed_bytes)) = observation { - return HostBudgetResolution::Resolved { - effective_bytes: env_override - .map(|requested| requested.min(observed_bytes)) - .unwrap_or(observed_bytes), - requested_bytes: env_override, + match crate::v1_rt::resolve_host_budget_join( + env_override, + cgroup_high, + cgroup_max, + v1, + darwin_physical, + ) { + crate::v1_rt::HostBudgetJoin::Resolved { + effective_bytes, + requested_bytes, + source, + observed_bytes, + } => HostBudgetResolution::Resolved { + effective_bytes, + requested_bytes, observation: HostBudgetObservation { - source, + source: match source { + crate::v1_rt::HostBudgetJoinSource::CgroupMemoryHigh { cgroup_dir } => { + HostBudgetSource::CgroupMemoryHigh { cgroup_dir } + } + crate::v1_rt::HostBudgetJoinSource::CgroupMemoryMax { cgroup_dir } => { + HostBudgetSource::CgroupMemoryMax { cgroup_dir } + } + crate::v1_rt::HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit { + cgroup_dir, + } => HostBudgetSource::CgroupV1HierarchicalMemoryLimit { cgroup_dir }, + crate::v1_rt::HostBudgetJoinSource::DarwinPhysicalMemory => { + HostBudgetSource::DarwinPhysicalMemory + } + }, bytes: observed_bytes, }, - }; - } - // Darwin only. `darwin_physical_memory_bytes` is `None` on every other target, so this - // arm cannot be reached on a kernel that has cgroups — which is precisely the wall - // `host_budget_source_admissible_as_bound_on_kernel` states: a host-shared reading may - // serve as the budget only where no private-limit mechanism exists. - if let Some(bytes) = darwin_physical { - return HostBudgetResolution::Resolved { - effective_bytes: env_override - .map(|requested| requested.min(bytes)) - .unwrap_or(bytes), - requested_bytes: env_override, - observation: HostBudgetObservation { - source: HostBudgetSource::DarwinPhysicalMemory, - bytes, - }, - }; - } - if let Some(requested_bytes) = env_override { - return HostBudgetResolution::DeclaredUnverified { + }, + crate::v1_rt::HostBudgetJoin::DeclaredUnverified { requested_bytes, - reason: "no observed private memory.high or memory.max verifies the executor allowance; the declaration is a planning request, not an enforced process limit".to_string(), - }; - } - HostBudgetResolution::Unreadable { - reason: host_budget_unreadable_reason(), + reason, + } => HostBudgetResolution::DeclaredUnverified { + requested_bytes, + reason, + }, + crate::v1_rt::HostBudgetJoin::Unreadable { reason } => { + HostBudgetResolution::Unreadable { reason } + } } } -/// The host memory planning ceiling, as a typed resolution. It does not cap RSS. Single authority shared -/// by the MemoryGovernor (which SCHEDULES against it), the typed-module cache cap (which -/// bounds an estimated ENTRY COUNT with it) and the P4 realize advisory (which PREDICTS against it) — no -/// consumer may re-read a partial version of this precedence (§3 single authority). +/// The host memory planning ceiling, as a typed resolution. Live observations and +/// precedence are `v1_rt::resolve_host_budget_join`; this is the typed wrapper. pub fn read_host_budget_resolution() -> HostBudgetResolution { let env_override = std::env::var("GUNBC_MEMORY_BUDGET_BYTES") .ok() .and_then(|s| s.trim().parse::().ok()); - let cgroup_high = binding_high_cgroup_dir().and_then(|dir| { - read_cgroup_u64(&dir, "memory.high").map(|v| (dir.display().to_string(), v)) - }); - let cgroup_max = binding_cap_cgroup_dir().and_then(|dir| { - read_cgroup_u64(&dir, "memory.max").map(|v| (dir.display().to_string(), v)) + let cgroup_high = crate::v1_rt::host_budget_tightest_cgroup("memory.high"); + let cgroup_max = crate::v1_rt::host_budget_tightest_cgroup("memory.max"); + let cgroup_v1_limit = crate::v1_rt::host_budget_cgroup_v1().map(|(dir, v)| { + ( + dir, + match v { + crate::v1_rt::HostBudgetCgroupV1::Limited(bytes) => { + CgroupV1MemoryLimitValue::Limited(bytes) + } + crate::v1_rt::HostBudgetCgroupV1::Unlimited => CgroupV1MemoryLimitValue::Unlimited, + crate::v1_rt::HostBudgetCgroupV1::Unparseable(body) => { + CgroupV1MemoryLimitValue::Unparseable(body) + } + }, + ) }); resolve_host_budget( env_override, cgroup_high, cgroup_max, - read_cgroup_v1_hierarchical_limit(), - darwin_physical_memory_bytes(), + cgroup_v1_limit, + crate::v1_rt::host_budget_darwin_physical(), ) } @@ -1387,29 +1375,8 @@ pub fn tightest_cgroup_dir_for(limit_file: &str) -> Option { /// `tightest_cgroup_dir_for` over supplied `/proc/self/cgroup` content and unified mount root, /// so a fixture hierarchy exercises the same walk. pub fn tightest_cgroup_dir_under(self_cg: &str, root: &Path, limit_file: &str) -> Option { - let rel = self_cg - .lines() - .find_map(|l| l.strip_prefix("0::")) - .map(|p| p.trim().trim_start_matches('/').to_string())?; - let mut dir = root.join(&rel); - let mut best: Option<(u64, PathBuf)> = None; - loop { - if let Ok(s) = std::fs::read_to_string(dir.join(limit_file)) { - let s = s.trim(); - if s != "max" { - if let Ok(v) = s.parse::() { - let take = best.as_ref().map(|(cur, _)| v < *cur).unwrap_or(true); - if take { - best = Some((v, dir.clone())); - } - } - } - } - if dir == root || !dir.pop() { - break; - } - } - best.map(|(_, d)| d) + crate::v1_rt::host_budget_tightest_cgroup_under(self_cg, root, limit_file) + .map(|(dir, _)| PathBuf::from(dir)) } /// The process's own deepest (leaf) cgroup from `/proc/self/cgroup`. @@ -1453,41 +1420,13 @@ pub fn memory_pressure_some_avg10(content: &str) -> Option { /// mounted, the process has no line on it, or its path is outside the mounted subtree — each of /// which leaves the budget to the other sources or to `Unreadable`, never to a guess. pub fn cgroup_v1_memory_dir(self_cg: &str, mountinfo: &str) -> Option { - let (mount_root, mount_point) = mountinfo.lines().find_map(|line| { - let fields: Vec<&str> = line.split(' ').collect(); - let dash = fields.iter().position(|f| *f == "-")?; - let fstype = fields.get(dash + 1)?; - let super_opts = fields.get(dash + 3)?; - if *fstype == "cgroup" && super_opts.split(',').any(|o| o == "memory") { - Some((fields.get(3)?.to_string(), fields.get(4)?.to_string())) - } else { - None - } - })?; - let path = self_cg.lines().find_map(|l| { - let mut parts = l.splitn(3, ':'); - let (_id, controllers, path) = (parts.next()?, parts.next()?, parts.next()?); - controllers - .split(',') - .any(|c| c == "memory") - .then(|| path.trim().to_string()) - })?; - let rel = if mount_root == "/" { - path.as_str() - } else { - let rest = path.strip_prefix(mount_root.as_str())?; - if !(rest.is_empty() || rest.starts_with('/')) { - return None; - } - rest - }; - Some(Path::new(&mount_point).join(rel.trim_start_matches('/'))) + crate::v1_rt::host_budget_cgroup_v1_memory_dir(self_cg, mountinfo).map(PathBuf::from) } /// Mirror of `extdeps.linux.cgroup_v1_memory` `cgroup_v1_unlimited_bytes`: the kernel's /// PAGE_COUNTER_MAX (LONG_MAX / PAGE_SIZE) reported in bytes, so the sentinel follows the page size. pub fn cgroup_v1_unlimited_bytes(page_size: u64) -> u64 { - (i64::MAX as u64 / page_size) * page_size + crate::v1_rt::host_budget_cgroup_v1_unlimited_bytes(page_size) } /// Mirror of `extdeps.linux.cgroup_v1_memory` `CgroupV1MemoryLimitValue`: three states, because @@ -1504,20 +1443,14 @@ pub fn cgroup_v1_hierarchical_limit_from_stat( memory_stat: &str, page_size: u64, ) -> CgroupV1MemoryLimitValue { - let hits: Vec<&str> = memory_stat - .lines() - .filter_map(|l| l.trim().strip_prefix("hierarchical_memory_limit ")) - .collect(); - let [body] = hits.as_slice() else { - return CgroupV1MemoryLimitValue::Unparseable(memory_stat.to_string()); - }; - match body.trim().parse::() { - Ok(n) if n < 0 => CgroupV1MemoryLimitValue::Unparseable(body.to_string()), - Ok(n) if n >= cgroup_v1_unlimited_bytes(page_size) as i128 => { - CgroupV1MemoryLimitValue::Unlimited + match crate::v1_rt::host_budget_cgroup_v1_from_stat(memory_stat, page_size) { + crate::v1_rt::HostBudgetCgroupV1::Limited(bytes) => { + CgroupV1MemoryLimitValue::Limited(bytes) + } + crate::v1_rt::HostBudgetCgroupV1::Unlimited => CgroupV1MemoryLimitValue::Unlimited, + crate::v1_rt::HostBudgetCgroupV1::Unparseable(body) => { + CgroupV1MemoryLimitValue::Unparseable(body) } - Ok(n) => CgroupV1MemoryLimitValue::Limited(n as u64), - Err(_) => CgroupV1MemoryLimitValue::Unparseable(body.to_string()), } } @@ -1531,27 +1464,41 @@ pub fn cgroup_v1_hierarchical_limit_under( mountinfo: &str, page_size: u64, ) -> Option<(String, CgroupV1MemoryLimitValue)> { - let dir = cgroup_v1_memory_dir(self_cg, mountinfo)?; - let dir = fs_root.join(dir.strip_prefix("/").unwrap_or(&dir)); - let value = match std::fs::read_to_string(dir.join("memory.stat")) { - Ok(stat) => cgroup_v1_hierarchical_limit_from_stat(&stat, page_size), - Err(e) => CgroupV1MemoryLimitValue::Unparseable(format!("memory.stat: {e}")), - }; - Some((dir.display().to_string(), value)) + crate::v1_rt::host_budget_cgroup_v1_under(fs_root, self_cg, mountinfo, page_size).map( + |(dir, v)| { + ( + dir, + match v { + crate::v1_rt::HostBudgetCgroupV1::Limited(bytes) => { + CgroupV1MemoryLimitValue::Limited(bytes) + } + crate::v1_rt::HostBudgetCgroupV1::Unlimited => { + CgroupV1MemoryLimitValue::Unlimited + } + crate::v1_rt::HostBudgetCgroupV1::Unparseable(body) => { + CgroupV1MemoryLimitValue::Unparseable(body) + } + }, + ) + }, + ) } pub fn read_cgroup_v1_hierarchical_limit() -> Option<(String, CgroupV1MemoryLimitValue)> { - let self_cg = std::fs::read_to_string("/proc/self/cgroup").ok()?; - let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?; - // SAFETY: sysconf has no preconditions. - let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; - if page_size <= 0 { - return Some(( - "/proc/self/mountinfo".to_string(), - CgroupV1MemoryLimitValue::Unparseable("sysconf(_SC_PAGESIZE) unreadable".to_string()), - )); - } - cgroup_v1_hierarchical_limit_under(Path::new("/"), &self_cg, &mountinfo, page_size as u64) + crate::v1_rt::host_budget_cgroup_v1().map(|(dir, v)| { + ( + dir, + match v { + crate::v1_rt::HostBudgetCgroupV1::Limited(bytes) => { + CgroupV1MemoryLimitValue::Limited(bytes) + } + crate::v1_rt::HostBudgetCgroupV1::Unlimited => CgroupV1MemoryLimitValue::Unlimited, + crate::v1_rt::HostBudgetCgroupV1::Unparseable(body) => { + CgroupV1MemoryLimitValue::Unparseable(body) + } + }, + ) + }) } pub fn read_cgroup_u64(dir: &Path, file: &str) -> Option { @@ -1575,32 +1522,8 @@ pub fn read_cgroup_raw(dir: &Path, file: &str) -> Option { /// /// Exists because the governor previously had NO source on Darwin and fell back to the most /// permissive cap it could name; macOS's memory facts were never asked for. -#[cfg(target_os = "macos")] -pub fn darwin_physical_memory_bytes() -> Option { - let name = c"hw.memsize"; - let mut value: u64 = 0; - let mut len: libc::size_t = std::mem::size_of::() as libc::size_t; - // SAFETY: `name` is a NUL-terminated literal, `value`/`len` are live locals sized to - // match, and `newp`/`newlen` are null/0 for a read-only query per sysctl(3). - let rc = unsafe { - libc::sysctlbyname( - name.as_ptr(), - (&mut value as *mut u64).cast::(), - &mut len, - std::ptr::null_mut(), - 0, - ) - }; - if rc == 0 && value > 0 { - Some(value) - } else { - None - } -} - -#[cfg(not(target_os = "macos"))] pub fn darwin_physical_memory_bytes() -> Option { - None + crate::v1_rt::host_budget_darwin_physical() } /// The bind request an executor may carry, read from `GUNBC_BIND_MEMORY_CGROUP_BYTES`. diff --git a/src/v1/stage0/src/v1_compiler_emit.rs b/src/v1/stage0/src/v1_compiler_emit.rs index 7700517388f..99ba0f8df7b 100644 --- a/src/v1/stage0/src/v1_compiler_emit.rs +++ b/src/v1/stage0/src/v1_compiler_emit.rs @@ -21,6 +21,8 @@ pub use crate::std_coercion::TypeDeclarationProvenance; use crate::std_coercion::TypeDeclarationProvenance::DeclarationIdentityAbsent; pub use crate::std_coercion::TypeRealizationDecision; use crate::std_coercion::TypeRealizationDecision::*; +pub use crate::std_decl_ref::DeclarationRef; +pub use crate::std_decl_ref::{decl_ref, declaration_ref_eq}; use crate::std_induction::SubValueRelation::SubValueUnknown; pub use crate::std_induction::{InductiveField, SubValueRelation}; pub use crate::std_occurrence_identity::occurrence_id_eq; @@ -55,6 +57,7 @@ pub use crate::v1_compiler_emit_core_support::{EmitResult, TestProjection}; pub use crate::v1_compiler_infer::InferScope; pub use crate::v1_compiler_infer::{ build_params_scope, call_param_caller_labels, extend_scope, is_where_refinement_type, + resolved_type_name, }; use crate::v1_compiler_infer_emit_info::DataVariantWireSpelling::{ DataVariantBareString, DataVariantInternalTagged, DataVariantSpellingRefused, @@ -62,9 +65,14 @@ use crate::v1_compiler_infer_emit_info::DataVariantWireSpelling::{ }; pub use crate::v1_compiler_infer_emit_info::{DataVariantWireSpelling, EmitGraphInfo, TypeSummary}; use crate::v1_compiler_infer_env::GlobalBareLookupState::*; +use crate::v1_compiler_infer_env::TypeReferenceDeclarationReading::TypeReferenceNamesDeclaration; pub use crate::v1_compiler_infer_env::UnitVariantContribution; -pub use crate::v1_compiler_infer_env::{authored_name, empty_symbol_index, lookup_type_for}; -pub use crate::v1_compiler_infer_env::{GlobalBareLookupState, TypeBinding, TypeEnv}; +pub use crate::v1_compiler_infer_env::{ + authored_name, empty_symbol_index, lookup_type_for, type_reference_declaration_reading, +}; +pub use crate::v1_compiler_infer_env::{ + GlobalBareLookupState, TypeBinding, TypeEnv, TypeReferenceDeclarationReading, +}; pub use crate::v1_compiler_infer_items::{item_is_effectful_callee, item_resource_names}; pub use crate::v1_compiler_infer_items::{ItemInfo, ResolvedGraph, TypedModule}; pub use crate::v1_compiler_infer_lookup::lookup_func_sig; @@ -110,7 +118,7 @@ use crate::v1_std_core::CallSemantics::ResolvedDirectCallSemantics; use crate::v1_std_core::CallTargetIdentity::{ CallableTargetUndetermined, LocallyBoundCall, RuntimePrimitiveCall, SourceDeclarationCall, }; -use crate::v1_std_core::Cardinality::CardOptional; +use crate::v1_std_core::Cardinality::{CardOptional, Required}; use crate::v1_std_core::CompilerDiagnostic::TransportEmissionNotModeled; use crate::v1_std_core::Connective::{Arrow, Conj, Disj, NoConnective}; use crate::v1_std_core::ExprData::{ @@ -2827,6 +2835,7 @@ pub struct ShellResultField { pub enum ShellEmissionRefusal { ShellOutputKeyNotModeled { key: String }, ShellChannelNotRealizedByTarget { key: String, target_name: String }, + ShellCapturePolicyInputAbsent { key: String }, } impl ShellEmissionRefusal { pub fn key(&self) -> String { @@ -2835,6 +2844,7 @@ impl ShellEmissionRefusal { ShellEmissionRefusal::ShellChannelNotRealizedByTarget { key: __val, .. } => { __val.clone() } + ShellEmissionRefusal::ShellCapturePolicyInputAbsent { key: __val, .. } => __val.clone(), } } } @@ -2891,12 +2901,27 @@ pub fn shell_result_channel_key(c: ShellResultChannel) -> String { } } -pub fn shell_channel_realized_by_target(c: ShellResultChannel, target: RenderTarget) -> bool { +pub fn shell_channel_is_capture_accounting(c: ShellResultChannel) -> bool { match c.clone() { - ShellResultChannel::ShellChanStderrTruncated => false, - ShellResultChannel::ShellChanStderrTotalBytes => false, - ShellResultChannel::ShellChanStderrRetainedBytes => false, - _ => target_renders_shell_transport(target.clone()), + ShellResultChannel::ShellChanStderrTruncated => true, + ShellResultChannel::ShellChanStderrTotalBytes => true, + ShellResultChannel::ShellChanStderrRetainedBytes => true, + _ => false, + } +} + +pub fn shell_channel_realized_by_target(c: ShellResultChannel, target: RenderTarget) -> bool { + if shell_channel_is_capture_accounting(c.clone()) { + target_is_rust(target.clone()) + } else { + target_renders_shell_transport(target.clone()) + } +} + +pub fn target_is_rust(target: RenderTarget) -> bool { + match target.clone() { + RenderTarget::Rust => true, + _ => false, } } @@ -2911,6 +2936,7 @@ pub fn shell_emission_refusal_fact(refusal: Rc) -> String match (*refusal.clone()).clone() { ShellEmissionRefusal::ShellOutputKeyNotModeled { key: k, .. } => v1_rt::concat(v1_rt::concat("shell transport output key '".to_string(), k.clone()), "' has no modeled channel -- the modeled channels are stdout, stderr, exit_success, success, exists, exit_code, stdout_lines, stderr_truncated, stderr_total_bytes and stderr_retained_bytes".to_string()), ShellEmissionRefusal::ShellChannelNotRealizedByTarget { key: k, target_name: tn, .. } => v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat("shell output channel '".to_string(), k.clone()), "' is a modeled channel that target ".to_string()), tn.clone()), " cannot realize -- the emitted realization implements no stderr capture policy, so answering it would assert that the declared policy ran".to_string()), + ShellEmissionRefusal::ShellCapturePolicyInputAbsent { key: k, .. } => v1_rt::concat(v1_rt::concat("shell output channel '".to_string(), k.clone()), "' is a capture-accounting channel and this operation does not declare a required scalar stderr_capture: WitnessStderrCapturePolicy -- answering it would apply a policy nobody declared".to_string()), } } @@ -5407,6 +5433,62 @@ pub fn file_operation_has_content_input( ) } +pub fn required_stderr_capture_policy_declaration() -> Rc { + crate::std_decl_ref::decl_ref( + "std.shell_stream_capture".to_string(), + "WitnessStderrCapturePolicy".to_string(), + ) +} + +pub fn param_type_is_stderr_capture_policy_declaration(p: Rc, env: Rc) -> bool { + match (*crate::v1_compiler_infer_env::type_reference_declaration_reading( + crate::v1_std_core::param_node_type_expr(p.clone()), + env.source_indices.clone(), + env.clone(), + )) + .clone() + { + TypeReferenceDeclarationReading::TypeReferenceNamesDeclaration { + declaration: d, .. + } => crate::std_decl_ref::declaration_ref_eq( + d.clone(), + required_stderr_capture_policy_declaration(), + ), + _ => false, + } +} + +pub fn param_is_required_stderr_capture_policy(p: Rc, env: Rc) -> bool { + { + let ty = crate::v1_std_core::param_node_type_expr(p.clone()); + (((((crate::v1_std_core::param_node_name_at(p.clone(), env.source_indices.clone()) + == "stderr_capture".to_string()) + && (p.return_cardinality.clone() == Cardinality::Required)) + && (ty.return_cardinality.clone() == Cardinality::Required)) + && !crate::v1_compiler_infer_types::node_is_collection( + ty.clone(), + env.source_indices.clone(), + )) + && param_type_is_stderr_capture_policy_declaration(p.clone(), env.clone())) + } +} + +pub fn operation_declares_required_stderr_capture_policy( + op_node: Rc, + env: Rc, +) -> bool { + { + let mut __found = false; + for p in op_node.params.clone().iter().cloned() { + if param_is_required_stderr_capture_policy(p.clone(), env.clone()) { + __found = true; + break; + } + } + __found + } +} + pub fn is_modeled_file_output_channel(key: String) -> bool { match file_result_channel_of_key(key.clone()) { Some(_) => true, @@ -5732,21 +5814,35 @@ match crate::v1_std_core::classify_transport(t.clone(), si.clone()) { span: ch.span.clone(), }), module_name.clone())]) }, - Some(c) => if shell_channel_realized_by_target(c.clone(), target.clone()) { - Rc::new(vec![]) - } else { + Some(c) => if ((shell_channel_is_capture_accounting(c.clone()) && target_is_rust(target.clone())) && !operation_declares_required_stderr_capture_policy(op_node.clone(), env.clone())) { Rc::new(vec![crate::v1_std_core::make_error_node(Rc::new(CompilerDiagnostic::TransportEmissionNotModeled { transport_kind: "shell".to_string(), service: service_name.clone(), operation: crate::v1_compiler_infer_env::authored_name(env.clone(), op_node.clone()), declaring_module: module_name.clone(), target: render_target_name(target.clone()), + missing_realization_fact: shell_emission_refusal_fact(Rc::new(ShellEmissionRefusal::ShellCapturePolicyInputAbsent { + key: shell_result_channel_key(c.clone()), +})), + span: ch.span.clone(), +}), module_name.clone())]) + } else { + if shell_channel_realized_by_target(c.clone(), target.clone()) { + Rc::new(vec![]) + } else { + Rc::new(vec![crate::v1_std_core::make_error_node(Rc::new(CompilerDiagnostic::TransportEmissionNotModeled { + transport_kind: "shell".to_string(), + service: service_name.clone(), + operation: crate::v1_compiler_infer_env::authored_name(env.clone(), op_node.clone()), + declaring_module: module_name.clone(), + target: render_target_name(target.clone()), missing_realization_fact: shell_emission_refusal_fact(Rc::new(ShellEmissionRefusal::ShellChannelNotRealizedByTarget { key: shell_result_channel_key(c.clone()), target_name: render_target_name(target.clone()), })), span: ch.span.clone(), }), module_name.clone())]) + } }, }).iter().cloned()); } __result }), _ => Rc::new(vec![]), diff --git a/src/v1/stage0/src/v1_compiler_emit_rust.rs b/src/v1/stage0/src/v1_compiler_emit_rust.rs index 052115df96b..58e9b4913f3 100644 --- a/src/v1/stage0/src/v1_compiler_emit_rust.rs +++ b/src/v1/stage0/src/v1_compiler_emit_rust.rs @@ -178,7 +178,9 @@ use crate::v1_compiler_emit::FileVerb::{ FileDelete, FileLinkCreateNew, FileList, FileRead, FileWrite, FileWriteCreateNew, FileWriteCreateNewWithMode, FileWriteOwnerOnly, }; -use crate::v1_compiler_emit::ShellEmissionRefusal::ShellChannelNotRealizedByTarget; +use crate::v1_compiler_emit::ShellEmissionRefusal::{ + ShellCapturePolicyInputAbsent, ShellChannelNotRealizedByTarget, +}; use crate::v1_compiler_emit::ShellResultChannel::{ ShellChanExitCode, ShellChanExitSuccess, ShellChanStderr, ShellChanStderrRetainedBytes, ShellChanStderrTotalBytes, ShellChanStderrTruncated, ShellChanStdout, ShellChanStdoutLines, @@ -193,10 +195,11 @@ pub use crate::v1_compiler_emit::{ emit_typed_if_shared, emit_typed_let_shared, emit_unary_op, escape_rust_interp_text, extract_modifier_names, has_nested_records_node, has_service_items, is_null_coalesce, is_self_recursive, is_tco_eligible, keyed_container_has_target_inhabitant, - lookup_item_by_identity, module_emit_scope, order_typed_call_args_from_semantics, - render_node_type, render_tuple_parts, rust_literal_for_pattern, scope_after_expr, - seed_bindings, service_fallback_transport, service_field_ctors, service_field_decls, - shared_tco_reassign, shell_emission_refusal_fact, shell_result_channel_key, + lookup_item_by_identity, module_emit_scope, operation_declares_required_stderr_capture_policy, + order_typed_call_args_from_semantics, render_node_type, render_tuple_parts, + rust_literal_for_pattern, scope_after_expr, seed_bindings, service_fallback_transport, + service_field_ctors, service_field_decls, shared_tco_reassign, + shell_channel_is_capture_accounting, shell_emission_refusal_fact, shell_result_channel_key, tco_loop_iteration_lets, tco_loop_slot_name, tco_reassign_core, transport_binding_refusal_fact, }; pub use crate::v1_compiler_emit::{ @@ -37468,6 +37471,7 @@ pub fn emit_transport_call( rsf.clone(), op_node.clone(), source_indices.clone(), + env.clone(), ), BoundOperation::FileBound { verb: v, @@ -38823,7 +38827,7 @@ pub fn emit_exit_code_handling( v1_rt::concat( v1_rt::concat( "\n _ => { ".to_string(), - shell_stderr_binding_line(), + shell_error_stderr_binding(result_fields.clone()), ), " ".to_string(), ), @@ -38885,7 +38889,7 @@ pub fn emit_exit_arm( v1_rt::concat(" ".to_string(), pattern.clone()), " => { ".to_string(), ), - shell_stderr_binding_line(), + shell_error_stderr_binding(result_fields.clone()), ), " ".to_string(), ), @@ -38905,6 +38909,7 @@ pub fn emit_shell_call( result_fields: Rc>>, op_node: Rc, source_indices: Rc>>, + env: Rc, ) -> String { { let argv = transport.children.clone(); @@ -38959,7 +38964,8 @@ pub fn emit_shell_call( Some(_) => true, std::option::Option::None => false, }; - let let_kw = if has_stdin.clone() { + let needs_capture = shell_needs_capture_accounting(result_fields.clone()); + let let_kw = if (has_stdin.clone() || needs_capture.clone()) { "let mut output".to_string() } else { "let output".to_string() @@ -39078,79 +39084,136 @@ pub fn emit_shell_call( __result }); let wd_line = " .current_dir(self.working_dir.as_deref().unwrap_or(\".\"))".to_string(); - if has_stdin.clone() { + let return_line = emit_exit_code_handling( + op_node.clone(), + result_fields.clone(), + source_indices.clone(), + ); + if needs_capture.clone() { { - let stdin_expr = - crate::v1_std_core::transport_stdin(transport.clone(), source_indices.clone()) + let policy_bind = emit_shell_stderr_policy_binding(op_node.clone(), env.clone()); + let spawn_line = " .stdin(std::process::Stdio::piped())\n .stdout(std::process::Stdio::piped())\n .stderr(std::process::Stdio::piped())".to_string(); + let spawn_exec = " .spawn()?;".to_string(); + let stdin_thread = if has_stdin.clone() { + { + let stdin_expr = crate::v1_std_core::transport_stdin( + transport.clone(), + source_indices.clone(), + ) .clone() .unwrap(); - let stdin_var = match (*stdin_expr.expr_data.clone()).clone() { - ExprData::ExprVar { - binding_kind: _, .. - } => crate::v1_compiler_emit::emit_ident( - crate::v1_std_core::expr_var_name_at( - stdin_expr.clone(), - source_indices.clone(), - ), - RenderTarget::Rust, - ), - _ => crate::v1_compiler_emit::emit_simple_expr( - stdin_expr.clone(), - RenderTarget::Rust, - source_indices.clone(), - ), + let stdin_var = match (*stdin_expr.expr_data.clone()).clone() { + ExprData::ExprVar { + binding_kind: _, .. + } => crate::v1_compiler_emit::emit_ident( + crate::v1_std_core::expr_var_name_at( + stdin_expr.clone(), + source_indices.clone(), + ), + RenderTarget::Rust, + ), + _ => crate::v1_compiler_emit::emit_simple_expr( + stdin_expr.clone(), + RenderTarget::Rust, + source_indices.clone(), + ), + }; + v1_rt::concat(v1_rt::concat("let mut stdin_pipe = output.stdin.take();\nlet __stdin_thread = std::thread::spawn(move || -> std::io::Result<()> {\n use std::io::Write;\n if let Some(mut stdin) = stdin_pipe {\n stdin.write_all(".to_string(), stdin_var.clone()), ".as_bytes())?;\n }\n Ok(())\n});\n".to_string()) + } + } else { + "let mut stdin_pipe = output.stdin.take();\nlet __stdin_thread = std::thread::spawn(move || -> std::io::Result<()> {\n drop(stdin_pipe);\n Ok(())\n});\n".to_string() }; - let spawn_line = " .stdin(std::process::Stdio::piped())\n .stdout(std::process::Stdio::piped())\n .stderr(std::process::Stdio::piped())".to_string(); - let spawn_exec = " .spawn()?;".to_string(); - let write_block = v1_rt::concat(v1_rt::concat("{\n use std::io::Write;\n if let Some(mut stdin) = output.stdin.take() {\n stdin.write_all(".to_string(), stdin_var.clone()), ".as_bytes())?;\n }\n}".to_string()); - let wait_line = "let output = output.wait_with_output()?;".to_string(); - let check_line = - "let stdout = String::from_utf8_lossy(&output.stdout).to_string();".to_string(); - let return_line = emit_exit_code_handling( - op_node.clone(), - result_fields.clone(), - source_indices.clone(), + let capture_lines = v1_rt::concat( + v1_rt::concat(shell_capture_drain_start(), stdin_thread.clone()), + shell_capture_join_project(), ); - let all_lines = v1_rt::concat( + v1_rt::concat( v1_rt::concat( - v1_rt::concat(Rc::new(vec![cmd_line.clone()]), arg_lines.clone()), + v1_rt::concat( + Rc::new(vec![policy_bind.clone(), cmd_line.clone()]), + arg_lines.clone(), + ), env_lines.clone(), ), Rc::new(vec![ wd_line.clone(), spawn_line.clone(), spawn_exec.clone(), - write_block.clone(), - wait_line.clone(), - check_line.clone(), + capture_lines.clone(), return_line.clone(), ]), - ); - all_lines.clone().join(&"\n".to_string()) + ) + .join(&"\n".to_string()) } } else { - { - let output_line = " .output()?;".to_string(); - let check_line = - "let stdout = String::from_utf8_lossy(&output.stdout).to_string();".to_string(); - let return_line = emit_exit_code_handling( - op_node.clone(), - result_fields.clone(), - source_indices.clone(), - ); - let all_lines = v1_rt::concat( + if has_stdin.clone() { + { + let stdin_expr = crate::v1_std_core::transport_stdin( + transport.clone(), + source_indices.clone(), + ) + .clone() + .unwrap(); + let stdin_var = match (*stdin_expr.expr_data.clone()).clone() { + ExprData::ExprVar { + binding_kind: _, .. + } => crate::v1_compiler_emit::emit_ident( + crate::v1_std_core::expr_var_name_at( + stdin_expr.clone(), + source_indices.clone(), + ), + RenderTarget::Rust, + ), + _ => crate::v1_compiler_emit::emit_simple_expr( + stdin_expr.clone(), + RenderTarget::Rust, + source_indices.clone(), + ), + }; + let spawn_line = " .stdin(std::process::Stdio::piped())\n .stdout(std::process::Stdio::piped())\n .stderr(std::process::Stdio::piped())".to_string(); + let spawn_exec = " .spawn()?;".to_string(); + let write_block = v1_rt::concat(v1_rt::concat("{\n use std::io::Write;\n if let Some(mut stdin) = output.stdin.take() {\n stdin.write_all(".to_string(), stdin_var.clone()), ".as_bytes())?;\n }\n}".to_string()); + let wait_line = "let output = output.wait_with_output()?;".to_string(); + let check_line = + "let stdout = String::from_utf8_lossy(&output.stdout).to_string();" + .to_string(); v1_rt::concat( - v1_rt::concat(Rc::new(vec![cmd_line.clone()]), arg_lines.clone()), - env_lines.clone(), - ), - Rc::new(vec![ - wd_line.clone(), - output_line.clone(), - check_line.clone(), - return_line.clone(), - ]), - ); - all_lines.clone().join(&"\n".to_string()) + v1_rt::concat( + v1_rt::concat(Rc::new(vec![cmd_line.clone()]), arg_lines.clone()), + env_lines.clone(), + ), + Rc::new(vec![ + wd_line.clone(), + spawn_line.clone(), + spawn_exec.clone(), + write_block.clone(), + wait_line.clone(), + check_line.clone(), + return_line.clone(), + ]), + ) + .join(&"\n".to_string()) + } + } else { + { + let output_line = " .output()?;".to_string(); + let check_line = + "let stdout = String::from_utf8_lossy(&output.stdout).to_string();" + .to_string(); + v1_rt::concat( + v1_rt::concat( + v1_rt::concat(Rc::new(vec![cmd_line.clone()]), arg_lines.clone()), + env_lines.clone(), + ), + Rc::new(vec![ + wd_line.clone(), + output_line.clone(), + check_line.clone(), + return_line.clone(), + ]), + ) + .join(&"\n".to_string()) + } } } } @@ -39334,6 +39397,66 @@ pub fn shell_stderr_binding_line() -> String { CACHED.with(|c: &String| c.clone()) } +pub fn shell_needs_capture_accounting(result_fields: Rc>>) -> bool { + { + let mut __found = false; + for f in result_fields.iter().cloned() { + if crate::v1_compiler_emit::shell_channel_is_capture_accounting(f.channel.clone()) { + __found = true; + break; + } + } + __found + } +} + +pub fn shell_error_stderr_binding(result_fields: Rc>>) -> String { + if shell_needs_capture_accounting(result_fields.clone()) { + "".to_string() + } else { + v1_rt::concat(shell_stderr_binding_line(), " ".to_string()) + } +} + +pub fn emit_shell_stderr_policy_binding(op_node: Rc, env: Rc) -> String { + if crate::v1_compiler_emit::operation_declares_required_stderr_capture_policy( + op_node.clone(), + env.clone(), + ) { + v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat("let (__stderr_complete_limit, __stderr_complete_source): (Option, String) = match &*stderr_capture {\n".to_string(), " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::Complete => {\n".to_string()), " match v1_rt::read_host_budget_bytes() {\n".to_string()), " (Some(n), source) => (Some(n as usize), source),\n".to_string()), " (None, source) => return Err(format!(\"WitnessStderrCaptureCompleteBudgetUnreadable: Complete stderr capture requires the active host budget authority ({})\", source).into()),\n".to_string()), " }\n".to_string()), " }\n".to_string()), " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::BoundedTail { bytes } => {\n".to_string()), " let n = crate::std_measure::byte_size_count(bytes.clone());\n".to_string()), " if n < 0 { return Err(\"WitnessStderrCapturePolicy.BoundedTail bytes must be non-negative\".into()); }\n".to_string()), " (None, String::new())\n".to_string()), " }\n".to_string()), "};\n".to_string()), "let __stderr_tail_bytes: usize = match &*stderr_capture {\n".to_string()), " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::BoundedTail { bytes } => crate::std_measure::byte_size_count(bytes.clone()) as usize,\n".to_string()), " crate::std_shell_stream_capture::WitnessStderrCapturePolicy::Complete => 0,\n".to_string()), "};\n".to_string()) + } else { + v1_rt::concat( + v1_rt::concat( + "return Err(\"".to_string(), + crate::v1_compiler_emit::shell_emission_refusal_fact(Rc::new( + ShellEmissionRefusal::ShellCapturePolicyInputAbsent { + key: "stderr_truncated".to_string(), + }, + )), + ), + "\".into());\n".to_string(), + ) + } +} + +pub fn shell_capture_drain_start() -> String { + thread_local! { + static CACHED: String = { + "let mut stdout_pipe = output.stdout.take();\nlet mut stderr_pipe = output.stderr.take();\nlet stdout_thread = std::thread::spawn(move || -> std::io::Result> {\n let mut buf = Vec::new();\n if let Some(ref mut reader) = stdout_pipe {\n std::io::Read::read_to_end(reader, &mut buf)?;\n }\n Ok(buf)\n});\nlet stderr_thread = std::thread::spawn(move || -> std::io::Result<(Vec, u64, bool)> {\n let mut reader = match stderr_pipe {\n Some(r) => r,\n None => return Ok((Vec::new(), 0, false)),\n };\n let mut chunk = [0u8; 65536];\n let mut total: u64 = 0;\n if let Some(max_bytes) = __stderr_complete_limit {\n let mut retained = Vec::new();\n loop {\n let n = std::io::Read::read(&mut reader, &mut chunk)?;\n if n == 0 { break; }\n total += n as u64;\n if total <= max_bytes as u64 { retained.extend_from_slice(&chunk[..n]); }\n }\n let truncated = total > max_bytes as u64;\n return Ok((if truncated { Vec::new() } else { retained }, total, truncated));\n }\n let cap = __stderr_tail_bytes;\n let mut ring = Vec::with_capacity(cap);\n let mut start = 0usize;\n loop {\n let n = std::io::Read::read(&mut reader, &mut chunk)?;\n if n == 0 { break; }\n total += n as u64;\n for &b in &chunk[..n] {\n if cap == 0 { continue; }\n if ring.len() < cap {\n ring.push(b);\n } else {\n ring[start] = b;\n start = (start + 1) % cap;\n }\n }\n }\n let retained = if ring.len() < cap || cap == 0 {\n ring\n } else {\n let mut out = Vec::with_capacity(cap);\n for i in 0..cap { out.push(ring[(start + i) % cap]); }\n out\n };\n let retained_len = retained.len() as u64;\n Ok((retained, total, total > retained_len))\n});\n".to_string() + }; + } + CACHED.with(|c: &String| c.clone()) +} + +pub fn shell_capture_join_project() -> String { + thread_local! { + static CACHED: String = { + "let status = output.wait()?;\n__stdin_thread.join().map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, \"stdin write thread panicked\"))??;\nlet stdout_bytes = stdout_thread.join().map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, \"stdout drain thread panicked\"))??;\nlet (stderr_bytes, stderr_total_u64, stderr_truncated) = stderr_thread.join().map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, \"stderr drain thread panicked\"))??;\nif let Some(max_bytes) = __stderr_complete_limit {\n if stderr_total_u64 > max_bytes as u64 {\n return Err(format!(\"WitnessStderrCaptureCompleteBudgetExceeded: stderr total {} exceeds host budget {} ({})\", stderr_total_u64, max_bytes, __stderr_complete_source).into());\n }\n}\nlet stderr_total_bytes = stderr_total_u64 as i64;\nlet stderr_retained_bytes = stderr_bytes.len() as i64;\nlet stdout = String::from_utf8_lossy(&stdout_bytes).to_string();\nlet stderr = String::from_utf8_lossy(&stderr_bytes).trim_end().to_string();\nlet output = std::process::Output { status, stdout: stdout_bytes, stderr: stderr_bytes };\n".to_string() + }; + } + CACHED.with(|c: &String| c.clone()) +} + pub fn emit_shell_return(result_fields: Rc>>) -> String { v1_rt::concat( v1_rt::concat( @@ -39383,24 +39506,28 @@ pub fn shell_projection_value(result_fields: Rc>>) -> S } pub fn shell_stderr_prelude(result_fields: Rc>>) -> String { - { - let needs_stderr = { - let mut __found = false; - for f in result_fields.iter().cloned() { - if match f.channel.clone() { - ShellResultChannel::ShellChanStderr => true, - _ => false, - } { - __found = true; - break; + if shell_needs_capture_accounting(result_fields.clone()) { + "".to_string() + } else { + { + let needs_stderr = { + let mut __found = false; + for f in result_fields.iter().cloned() { + if match f.channel.clone() { + ShellResultChannel::ShellChanStderr => true, + _ => false, + } { + __found = true; + break; + } } + __found + }; + if needs_stderr.clone() { + v1_rt::concat(shell_stderr_binding_line(), "\n".to_string()) + } else { + "".to_string() } - __found - }; - if needs_stderr.clone() { - v1_rt::concat(shell_stderr_binding_line(), "\n".to_string()) - } else { - "".to_string() } } } @@ -39418,15 +39545,9 @@ pub fn emit_shell_channel_expr(channel: ShellResultChannel, is_optional: bool) - "stdout.lines().filter(|l| !l.is_empty()).map(|l| l.trim().to_string()).collect()" .to_string(), ), - ShellResultChannel::ShellChanStderrTruncated => { - emit_unrealizable_shell_channel(channel.clone()) - } - ShellResultChannel::ShellChanStderrTotalBytes => { - emit_unrealizable_shell_channel(channel.clone()) - } - ShellResultChannel::ShellChanStderrRetainedBytes => { - emit_unrealizable_shell_channel(channel.clone()) - } + ShellResultChannel::ShellChanStderrTruncated => "stderr_truncated".to_string(), + ShellResultChannel::ShellChanStderrTotalBytes => "stderr_total_bytes".to_string(), + ShellResultChannel::ShellChanStderrRetainedBytes => "stderr_retained_bytes".to_string(), }; if is_optional.clone() { v1_rt::concat( diff --git a/src/v1/stage0/src/v1_compiler_runtime_rust.rs b/src/v1/stage0/src/v1_compiler_runtime_rust.rs index 38d49cab731..0cb88422e52 100644 --- a/src/v1/stage0/src/v1_compiler_runtime_rust.rs +++ b/src/v1/stage0/src/v1_compiler_runtime_rust.rs @@ -135,39 +135,47 @@ pub fn rust_runtime_source() -> String { v1_rt::concat( v1_rt::concat( v1_rt::concat( - rt_header(), - rt_text_lookup_work_counter(), + v1_rt::concat( + rt_header(), + rt_text_lookup_work_counter(), + ), + rt_resolution_silent_pick_telemetry( + ), ), - rt_resolution_silent_pick_telemetry(), + rt_concat_trait(), ), - rt_concat_trait(), + rt_string_ops(), ), - rt_string_ops(), + rt_string_carrier(), ), - rt_string_carrier(), + rt_collection_ops(), ), - rt_collection_ops(), + rt_rc_container_ops(), ), - rt_rc_container_ops(), + rt_scanner_ops(), ), - rt_scanner_ops(), + rt_unicode_ops(), ), - rt_unicode_ops(), + rt_freemonoid_host_ops(), ), - rt_freemonoid_host_ops(), + rt_hash_ops(), ), - rt_hash_ops(), + rt_filesystem(), ), - rt_filesystem(), + rt_checked_int_ops(), ), - rt_checked_int_ops(), + rt_realization_measurement(), ), - rt_realization_measurement(), + rt_accelerator_demo_kernel(), ), - rt_accelerator_demo_kernel(), + rt_host_budget(), ) } +pub fn rt_host_budget() -> String { + v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat("/// The one host-budget precedence. `read_host_budget_bytes` and\n".to_string(), "/// `memory_governor::resolve_host_budget` both call `resolve_host_budget_join`.\n".to_string()), "#[derive(Clone, Debug, PartialEq, Eq)]\n".to_string()), "pub enum HostBudgetJoinSource {\n".to_string()), " CgroupMemoryHigh { cgroup_dir: String },\n".to_string()), " CgroupMemoryMax { cgroup_dir: String },\n".to_string()), " CgroupV1HierarchicalMemoryLimit { cgroup_dir: String },\n".to_string()), " DarwinPhysicalMemory,\n".to_string()), "}\n\n".to_string()), "impl HostBudgetJoinSource {\n".to_string()), " pub fn label(&self) -> String {\n".to_string()), " match self {\n".to_string()), " HostBudgetJoinSource::CgroupMemoryHigh { cgroup_dir } => format!(\"cgroup memory.high ({})\", cgroup_dir),\n".to_string()), " HostBudgetJoinSource::CgroupMemoryMax { cgroup_dir } => format!(\"cgroup memory.max ({})\", cgroup_dir),\n".to_string()), " HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit { cgroup_dir } => format!(\"cgroup v1 memory.stat hierarchical_memory_limit ({})\", cgroup_dir),\n".to_string()), " HostBudgetJoinSource::DarwinPhysicalMemory => \"sysctl hw.memsize\".to_string(),\n".to_string()), " }\n".to_string()), " }\n".to_string()), "}\n\n".to_string()), "#[derive(Clone, Debug, PartialEq, Eq)]\n".to_string()), "pub enum HostBudgetJoin {\n".to_string()), " Resolved { effective_bytes: u64, requested_bytes: Option, source: HostBudgetJoinSource, observed_bytes: u64 },\n".to_string()), " DeclaredUnverified { requested_bytes: u64, reason: String },\n".to_string()), " Unreadable { reason: String },\n".to_string()), "}\n\n".to_string()), "impl HostBudgetJoin {\n".to_string()), " pub fn bytes(&self) -> Option {\n".to_string()), " match self {\n".to_string()), " HostBudgetJoin::Resolved { effective_bytes, .. } => Some(*effective_bytes),\n".to_string()), " HostBudgetJoin::DeclaredUnverified { requested_bytes, .. } => Some(*requested_bytes),\n".to_string()), " HostBudgetJoin::Unreadable { .. } => None,\n".to_string()), " }\n".to_string()), " }\n".to_string()), " pub fn label(&self) -> String {\n".to_string()), " match self {\n".to_string()), " HostBudgetJoin::Resolved { effective_bytes, requested_bytes, source, observed_bytes } => match requested_bytes {\n".to_string()), " Some(requested) => format!(\"effective planning minimum {} bytes (env request {}; observed {}={} bytes)\", effective_bytes, requested, source.label(), observed_bytes),\n".to_string()), " None => source.label(),\n".to_string()), " },\n".to_string()), " HostBudgetJoin::Unreadable { reason } => format!(\"unreadable: {}\", reason),\n".to_string()), " HostBudgetJoin::DeclaredUnverified { requested_bytes, reason } => format!(\"declared-unverified: env GUNBC_MEMORY_BUDGET_BYTES={}; {}\", requested_bytes, reason),\n".to_string()), " }\n".to_string()), " }\n".to_string()), "}\n\n".to_string()), "pub fn resolve_host_budget_join(env_override: Option, cgroup_high: Option<(String, u64)>, cgroup_max: Option<(String, u64)>, cgroup_v1_limit: Option<(String, HostBudgetCgroupV1)>, darwin_physical: Option) -> HostBudgetJoin {\n".to_string()), " let cgroup_v1_limit = match cgroup_v1_limit {\n".to_string()), " Some((dir, HostBudgetCgroupV1::Unparseable(body))) => {\n".to_string()), " return HostBudgetJoin::Unreadable { reason: format!(\"cgroup v1 memory hierarchy at {} holds this process but its hierarchical_memory_limit is unreadable ({}); a bound that may be the tightest cannot be replaced by another reading\", dir, body) };\n".to_string()), " }\n".to_string()), " Some((dir, HostBudgetCgroupV1::Limited(bytes))) => Some((dir, bytes)),\n".to_string()), " Some((_, HostBudgetCgroupV1::Unlimited)) | None => None,\n".to_string()), " };\n".to_string()), " let observation = [\n".to_string()), " cgroup_high.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupMemoryHigh { cgroup_dir }, b)),\n".to_string()), " cgroup_max.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupMemoryMax { cgroup_dir }, b)),\n".to_string()), " cgroup_v1_limit.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit { cgroup_dir }, b)),\n".to_string()), " ].into_iter().flatten().fold(None::<(HostBudgetJoinSource, u64)>, |best, cand| match best { Some(cur) if cur.1 <= cand.1 => Some(cur), _ => Some(cand) });\n".to_string()), " if let Some((source, observed_bytes)) = observation {\n".to_string()), " return HostBudgetJoin::Resolved { effective_bytes: env_override.map(|requested| requested.min(observed_bytes)).unwrap_or(observed_bytes), requested_bytes: env_override, source, observed_bytes };\n".to_string()), " }\n".to_string()), " if let Some(bytes) = darwin_physical {\n".to_string()), " return HostBudgetJoin::Resolved { effective_bytes: env_override.map(|requested| requested.min(bytes)).unwrap_or(bytes), requested_bytes: env_override, source: HostBudgetJoinSource::DarwinPhysicalMemory, observed_bytes: bytes };\n".to_string()), " }\n".to_string()), " if let Some(requested_bytes) = env_override {\n".to_string()), " return HostBudgetJoin::DeclaredUnverified { requested_bytes, reason: \"no observed private memory.high, memory.max or v1 hierarchical_memory_limit verifies the executor allowance; the declaration is a planning request, not an enforced process limit\".to_string() };\n".to_string()), " }\n".to_string()), " HostBudgetJoin::Unreadable { reason: format!(\"no cgroup memory.high, memory.max or v1 hierarchical_memory_limit binds this process and GUNBC_MEMORY_BUDGET_BYTES cannot verify one (target_os={}), so the planning allowance is UNKNOWN. Refusing rather than admitting against the widest signal available: a host-shared reading is a number about the MACHINE, not about this slot, and admitting against one is the rc=137 SIGKILL this arm exists to prevent (BuildBuddy receipt 2026-08-30, gunbc.host_budget_source host_budget_source_seed_mirror_disposition). The executor must expose an enforceable limit; GUNBC_MEMORY_BUDGET_BYTES may only request a lower planning ceiling.\", std::env::consts::OS) }\n".to_string()), "}\n\n".to_string()), "pub fn read_host_budget_bytes() -> (Option, String) {\n".to_string()), " let join = resolve_host_budget_join(\n".to_string()), " std::env::var(\"GUNBC_MEMORY_BUDGET_BYTES\").ok().and_then(|s| s.trim().parse::().ok()),\n".to_string()), " host_budget_tightest_cgroup(\"memory.high\"),\n".to_string()), " host_budget_tightest_cgroup(\"memory.max\"),\n".to_string()), " host_budget_cgroup_v1(),\n".to_string()), " host_budget_darwin_physical(),\n".to_string()), " );\n".to_string()), " (join.bytes(), join.label())\n".to_string()), "}\n\n".to_string()), "#[derive(Clone)]\n".to_string()), "pub enum HostBudgetCgroupV1 {\n".to_string()), " Limited(u64),\n".to_string()), " Unlimited,\n".to_string()), " Unparseable(String),\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_tightest_cgroup_under(self_cg: &str, root: &std::path::Path, limit_file: &str) -> Option<(String, u64)> {\n".to_string()), " let rel = self_cg\n".to_string()), " .lines()\n".to_string()), " .find_map(|l| l.strip_prefix(\"0::\"))\n".to_string()), " .map(|p| p.trim().trim_start_matches('/').to_string())?;\n".to_string()), " let mut dir = root.join(&rel);\n".to_string()), " let mut best: Option<(u64, std::path::PathBuf)> = None;\n".to_string()), " loop {\n".to_string()), " if let Ok(s) = std::fs::read_to_string(dir.join(limit_file)) {\n".to_string()), " let s = s.trim();\n".to_string()), " if s != \"max\" {\n".to_string()), " if let Ok(v) = s.parse::() {\n".to_string()), " let take = best.as_ref().map(|(cur, _)| v < *cur).unwrap_or(true);\n".to_string()), " if take {\n".to_string()), " best = Some((v, dir.clone()));\n".to_string()), " }\n".to_string()), " }\n".to_string()), " }\n".to_string()), " }\n".to_string()), " if dir == root || !dir.pop() {\n".to_string()), " break;\n".to_string()), " }\n".to_string()), " }\n".to_string()), " best.map(|(v, d)| (d.display().to_string(), v))\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_tightest_cgroup(limit_file: &str) -> Option<(String, u64)> {\n".to_string()), " let self_cg = std::fs::read_to_string(\"/proc/self/cgroup\").ok()?;\n".to_string()), " host_budget_tightest_cgroup_under(&self_cg, std::path::Path::new(\"/sys/fs/cgroup\"), limit_file)\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_cgroup_v1_unlimited_bytes(page_size: u64) -> u64 {\n".to_string()), " (i64::MAX as u64 / page_size) * page_size\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_cgroup_v1() -> Option<(String, HostBudgetCgroupV1)> {\n".to_string()), " let self_cg = std::fs::read_to_string(\"/proc/self/cgroup\").ok()?;\n".to_string()), " let mountinfo = std::fs::read_to_string(\"/proc/self/mountinfo\").ok()?;\n".to_string()), " let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };\n".to_string()), " if page_size <= 0 {\n".to_string()), " return Some((\n".to_string()), " \"/proc/self/mountinfo\".to_string(),\n".to_string()), " HostBudgetCgroupV1::Unparseable(\"sysconf(_SC_PAGESIZE) unreadable\".to_string()),\n".to_string()), " ));\n".to_string()), " }\n".to_string()), " host_budget_cgroup_v1_under(std::path::Path::new(\"/\"), &self_cg, &mountinfo, page_size as u64)\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_cgroup_v1_under(fs_root: &std::path::Path, self_cg: &str, mountinfo: &str, page_size: u64) -> Option<(String, HostBudgetCgroupV1)> {\n".to_string()), " let dir = host_budget_cgroup_v1_memory_dir(self_cg, mountinfo)?;\n".to_string()), " let dir_path = std::path::Path::new(&dir);\n".to_string()), " let joined = fs_root.join(dir_path.strip_prefix(\"/\").unwrap_or(dir_path));\n".to_string()), " let value = match std::fs::read_to_string(joined.join(\"memory.stat\")) {\n".to_string()), " Ok(stat) => host_budget_cgroup_v1_from_stat(&stat, page_size),\n".to_string()), " Err(e) => HostBudgetCgroupV1::Unparseable(format!(\"memory.stat: {}\", e)),\n".to_string()), " };\n".to_string()), " Some((joined.display().to_string(), value))\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_cgroup_v1_memory_dir(self_cg: &str, mountinfo: &str) -> Option {\n".to_string()), " let (mount_root, mount_point) = mountinfo.lines().find_map(|line| {\n".to_string()), " let fields: Vec<&str> = line.split(' ').collect();\n".to_string()), " let dash = fields.iter().position(|f| *f == \"-\")?;\n".to_string()), " let fstype = fields.get(dash + 1)?;\n".to_string()), " let super_opts = fields.get(dash + 3)?;\n".to_string()), " if *fstype == \"cgroup\" && super_opts.split(',').any(|o| o == \"memory\") {\n".to_string()), " Some((fields.get(3)?.to_string(), fields.get(4)?.to_string()))\n".to_string()), " } else {\n".to_string()), " None\n".to_string()), " }\n".to_string()), " })?;\n".to_string()), " let path = self_cg.lines().find_map(|l| {\n".to_string()), " let mut parts = l.splitn(3, ':');\n".to_string()), " let (_id, controllers, path) = (parts.next()?, parts.next()?, parts.next()?);\n".to_string()), " controllers.split(',').any(|c| c == \"memory\").then(|| path.trim().to_string())\n".to_string()), " })?;\n".to_string()), " let rel = if mount_root == \"/\" {\n".to_string()), " path.as_str()\n".to_string()), " } else {\n".to_string()), " let rest = path.strip_prefix(mount_root.as_str())?;\n".to_string()), " if !(rest.is_empty() || rest.starts_with('/')) { return None; }\n".to_string()), " rest\n".to_string()), " };\n".to_string()), " Some(std::path::Path::new(&mount_point).join(rel.trim_start_matches('/')).display().to_string())\n".to_string()), "}\n\n".to_string()), "pub fn host_budget_cgroup_v1_from_stat(memory_stat: &str, page_size: u64) -> HostBudgetCgroupV1 {\n".to_string()), " let hits: std::vec::Vec<&str> = memory_stat.lines().filter_map(|l| l.trim().strip_prefix(\"hierarchical_memory_limit \")).collect();\n".to_string()), " let [body] = hits.as_slice() else { return HostBudgetCgroupV1::Unparseable(memory_stat.to_string()); };\n".to_string()), " let unlimited = host_budget_cgroup_v1_unlimited_bytes(page_size);\n".to_string()), " match body.trim().parse::() {\n".to_string()), " Ok(n) if n < 0 => HostBudgetCgroupV1::Unparseable(body.to_string()),\n".to_string()), " Ok(n) if n >= unlimited as i128 => HostBudgetCgroupV1::Unlimited,\n".to_string()), " Ok(n) => HostBudgetCgroupV1::Limited(n as u64),\n".to_string()), " Err(_) => HostBudgetCgroupV1::Unparseable(body.to_string()),\n".to_string()), " }\n".to_string()), "}\n\n".to_string()), "/// Darwin `hw.memsize` via `sysctlbyname`. Authority: `extdeps.darwin.sysctl` `HwMemsize`.\n".to_string()), "pub fn host_budget_darwin_physical() -> Option {\n".to_string()), " #[cfg(target_os = \"macos\")]\n".to_string()), " {\n".to_string()), " let name = std::ffi::CStr::from_bytes_with_nul(b\"hw.memsize\\0\").ok()?;\n".to_string()), " let mut value: u64 = 0;\n".to_string()), " let mut len: libc::size_t = std::mem::size_of::() as libc::size_t;\n".to_string()), " let rc = unsafe { libc::sysctlbyname(name.as_ptr(), (&mut value as *mut u64).cast::(), &mut len, std::ptr::null_mut(), 0) };\n".to_string()), " if rc == 0 && value > 0 { Some(value) } else { None }\n".to_string()), " }\n".to_string()), " #[cfg(not(target_os = \"macos\"))]\n".to_string()), " { None }\n".to_string()), "}\n".to_string()) +} + pub fn rt_accelerator_demo_kernel() -> String { v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat(v1_rt::concat("fn int_relu(x: i64) -> i64 {\n".to_string(), " if x > 0 { x } else { 0 }\n".to_string()), "}\n\n".to_string()), "/// Opcode wire authority: `extdeps.languages.simd.kernel.relu_mul_add_wire_op_codes`.\n".to_string()), "/// Scaffold regen-sync: see `host_kernel_relu_mul_add_op_codes_scaffold_note` until feature:dag-kernel-realization-handler.\n".to_string()), "const RELU_MUL_ADD_OP_CODES: [i64; 3] = [1, 2, 3];\n\n".to_string()), "fn assert_relu_mul_add_op_codes(op_codes: &[i64], _handler: &str) {\n".to_string()), " if op_codes != RELU_MUL_ADD_OP_CODES {\n".to_string()), " panic!(\"unsupported op_codes in accelerator demo kernel\");\n".to_string()), " }\n".to_string()), "}\n\n".to_string()), "/// Host bridge: `.dag` passes `std.numerical_contract.FmaContractionPolicy`; interpreter maps to 0/1 until target_model emit.\n".to_string()), "pub fn contiguous_loop_elementwise_float_kernel(\n".to_string()), " op_codes: &[i64],\n".to_string()), " fma_contraction_policy: i64,\n".to_string()), " a: &[f64],\n".to_string()), " b: &[f64],\n".to_string()), " c: &[f64],\n".to_string()), ") -> Vec {\n".to_string()), " assert_relu_mul_add_op_codes(op_codes, \"contiguous_loop_elementwise_float_kernel\");\n".to_string()), " assert_eq!(a.len(), b.len());\n".to_string()), " assert_eq!(b.len(), c.len());\n".to_string()), " let mut out = Vec::with_capacity(a.len());\n".to_string()), " for i in 0..a.len() {\n".to_string()), " let elem = match fma_contraction_policy {\n".to_string()), " 0 => {\n".to_string()), " let prod = a[i] * b[i];\n".to_string()), " prod + c[i]\n".to_string()), " },\n".to_string()), " 1 => a[i].mul_add(b[i], c[i]),\n".to_string()), " _ => panic!(\"unknown fma_contraction_policy\"),\n".to_string()), " };\n".to_string()), " out.push(if elem > 0.0 { elem } else { 0.0 });\n".to_string()), " }\n".to_string()), " out\n".to_string()), "}\n\n".to_string()), "/// Integer oracle authority: `gunbc.accelerator_demo_eval` via interpreter `Int` binops (`*`/`+`).\n".to_string()), "/// Host kernel uses `wrapping_*` to mirror release interpreter overflow semantics; demo fixtures\n".to_string()), "/// stay in-range under `IntegerExact` — out-of-range inputs are outside the bit-exact bar.\n".to_string()), "pub fn contiguous_loop_elementwise_kernel(\n".to_string()), " op_codes: &[i64],\n".to_string()), " a: &[i64],\n".to_string()), " b: &[i64],\n".to_string()), " c: &[i64],\n".to_string()), ") -> Vec {\n".to_string()), " assert_relu_mul_add_op_codes(op_codes, \"contiguous_loop_elementwise_kernel\");\n".to_string()), " assert_eq!(a.len(), b.len());\n".to_string()), " assert_eq!(b.len(), c.len());\n".to_string()), " let mut out = Vec::with_capacity(a.len());\n".to_string()), " for i in 0..a.len() {\n".to_string()), " let tmp = a[i].wrapping_mul(b[i]).wrapping_add(c[i]);\n".to_string()), " out.push(int_relu(tmp));\n".to_string()), " }\n".to_string()), " out\n".to_string()), "}\n".to_string()) } diff --git a/src/v1/stage0/src/v1_interpreter.rs b/src/v1/stage0/src/v1_interpreter.rs index 6ed58544e37..db7b048d05c 100644 --- a/src/v1/stage0/src/v1_interpreter.rs +++ b/src/v1/stage0/src/v1_interpreter.rs @@ -18076,7 +18076,7 @@ fn dispatch_shell( let (budget, source) = crate::memory_governor::read_host_budget_bytes(); Some(budget.ok_or_else(|| InterpError::TypeError { msg: format!( - "WitnessStderrCaptureCompleteBudgetUnreadable: Complete stderr capture requires the active GUNBC_MEMORY_BUDGET_BYTES authority ({source})" + "WitnessStderrCaptureCompleteBudgetUnreadable: Complete stderr capture requires the active host budget authority ({source})" ), })? as usize) } diff --git a/src/v1/stage0/src/v1_rt.rs b/src/v1/stage0/src/v1_rt.rs index 7204701cdf4..fffaf7cad8c 100644 --- a/src/v1/stage0/src/v1_rt.rs +++ b/src/v1/stage0/src/v1_rt.rs @@ -1614,3 +1614,323 @@ pub fn contiguous_loop_elementwise_kernel( } out } +/// The one host-budget precedence. `read_host_budget_bytes` and +/// `memory_governor::resolve_host_budget` both call `resolve_host_budget_join`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HostBudgetJoinSource { + CgroupMemoryHigh { cgroup_dir: String }, + CgroupMemoryMax { cgroup_dir: String }, + CgroupV1HierarchicalMemoryLimit { cgroup_dir: String }, + DarwinPhysicalMemory, +} + +impl HostBudgetJoinSource { + pub fn label(&self) -> String { + match self { + HostBudgetJoinSource::CgroupMemoryHigh { cgroup_dir } => { + format!("cgroup memory.high ({})", cgroup_dir) + } + HostBudgetJoinSource::CgroupMemoryMax { cgroup_dir } => { + format!("cgroup memory.max ({})", cgroup_dir) + } + HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit { cgroup_dir } => format!( + "cgroup v1 memory.stat hierarchical_memory_limit ({})", + cgroup_dir + ), + HostBudgetJoinSource::DarwinPhysicalMemory => "sysctl hw.memsize".to_string(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HostBudgetJoin { + Resolved { + effective_bytes: u64, + requested_bytes: Option, + source: HostBudgetJoinSource, + observed_bytes: u64, + }, + DeclaredUnverified { + requested_bytes: u64, + reason: String, + }, + Unreadable { + reason: String, + }, +} + +impl HostBudgetJoin { + pub fn bytes(&self) -> Option { + match self { + HostBudgetJoin::Resolved { + effective_bytes, .. + } => Some(*effective_bytes), + HostBudgetJoin::DeclaredUnverified { + requested_bytes, .. + } => Some(*requested_bytes), + HostBudgetJoin::Unreadable { .. } => None, + } + } + pub fn label(&self) -> String { + match self { + HostBudgetJoin::Resolved { + effective_bytes, + requested_bytes, + source, + observed_bytes, + } => match requested_bytes { + Some(requested) => format!( + "effective planning minimum {} bytes (env request {}; observed {}={} bytes)", + effective_bytes, + requested, + source.label(), + observed_bytes + ), + None => source.label(), + }, + HostBudgetJoin::Unreadable { reason } => format!("unreadable: {}", reason), + HostBudgetJoin::DeclaredUnverified { + requested_bytes, + reason, + } => format!( + "declared-unverified: env GUNBC_MEMORY_BUDGET_BYTES={}; {}", + requested_bytes, reason + ), + } + } +} + +pub fn resolve_host_budget_join( + env_override: Option, + cgroup_high: Option<(String, u64)>, + cgroup_max: Option<(String, u64)>, + cgroup_v1_limit: Option<(String, HostBudgetCgroupV1)>, + darwin_physical: Option, +) -> HostBudgetJoin { + let cgroup_v1_limit = match cgroup_v1_limit { + Some((dir, HostBudgetCgroupV1::Unparseable(body))) => { + return HostBudgetJoin::Unreadable { reason: format!("cgroup v1 memory hierarchy at {} holds this process but its hierarchical_memory_limit is unreadable ({}); a bound that may be the tightest cannot be replaced by another reading", dir, body) }; + } + Some((dir, HostBudgetCgroupV1::Limited(bytes))) => Some((dir, bytes)), + Some((_, HostBudgetCgroupV1::Unlimited)) | None => None, + }; + let observation = [ + cgroup_high + .map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupMemoryHigh { cgroup_dir }, b)), + cgroup_max.map(|(cgroup_dir, b)| (HostBudgetJoinSource::CgroupMemoryMax { cgroup_dir }, b)), + cgroup_v1_limit.map(|(cgroup_dir, b)| { + ( + HostBudgetJoinSource::CgroupV1HierarchicalMemoryLimit { cgroup_dir }, + b, + ) + }), + ] + .into_iter() + .flatten() + .fold( + None::<(HostBudgetJoinSource, u64)>, + |best, cand| match best { + Some(cur) if cur.1 <= cand.1 => Some(cur), + _ => Some(cand), + }, + ); + if let Some((source, observed_bytes)) = observation { + return HostBudgetJoin::Resolved { + effective_bytes: env_override + .map(|requested| requested.min(observed_bytes)) + .unwrap_or(observed_bytes), + requested_bytes: env_override, + source, + observed_bytes, + }; + } + if let Some(bytes) = darwin_physical { + return HostBudgetJoin::Resolved { + effective_bytes: env_override + .map(|requested| requested.min(bytes)) + .unwrap_or(bytes), + requested_bytes: env_override, + source: HostBudgetJoinSource::DarwinPhysicalMemory, + observed_bytes: bytes, + }; + } + if let Some(requested_bytes) = env_override { + return HostBudgetJoin::DeclaredUnverified { requested_bytes, reason: "no observed private memory.high, memory.max or v1 hierarchical_memory_limit verifies the executor allowance; the declaration is a planning request, not an enforced process limit".to_string() }; + } + HostBudgetJoin::Unreadable { reason: format!("no cgroup memory.high, memory.max or v1 hierarchical_memory_limit binds this process and GUNBC_MEMORY_BUDGET_BYTES cannot verify one (target_os={}), so the planning allowance is UNKNOWN. Refusing rather than admitting against the widest signal available: a host-shared reading is a number about the MACHINE, not about this slot, and admitting against one is the rc=137 SIGKILL this arm exists to prevent (BuildBuddy receipt 2026-08-30, gunbc.host_budget_source host_budget_source_seed_mirror_disposition). The executor must expose an enforceable limit; GUNBC_MEMORY_BUDGET_BYTES may only request a lower planning ceiling.", std::env::consts::OS) } +} + +pub fn read_host_budget_bytes() -> (Option, String) { + let join = resolve_host_budget_join( + std::env::var("GUNBC_MEMORY_BUDGET_BYTES") + .ok() + .and_then(|s| s.trim().parse::().ok()), + host_budget_tightest_cgroup("memory.high"), + host_budget_tightest_cgroup("memory.max"), + host_budget_cgroup_v1(), + host_budget_darwin_physical(), + ); + (join.bytes(), join.label()) +} + +#[derive(Clone)] +pub enum HostBudgetCgroupV1 { + Limited(u64), + Unlimited, + Unparseable(String), +} + +pub fn host_budget_tightest_cgroup_under( + self_cg: &str, + root: &std::path::Path, + limit_file: &str, +) -> Option<(String, u64)> { + let rel = self_cg + .lines() + .find_map(|l| l.strip_prefix("0::")) + .map(|p| p.trim().trim_start_matches('/').to_string())?; + let mut dir = root.join(&rel); + let mut best: Option<(u64, std::path::PathBuf)> = None; + loop { + if let Ok(s) = std::fs::read_to_string(dir.join(limit_file)) { + let s = s.trim(); + if s != "max" { + if let Ok(v) = s.parse::() { + let take = best.as_ref().map(|(cur, _)| v < *cur).unwrap_or(true); + if take { + best = Some((v, dir.clone())); + } + } + } + } + if dir == root || !dir.pop() { + break; + } + } + best.map(|(v, d)| (d.display().to_string(), v)) +} + +pub fn host_budget_tightest_cgroup(limit_file: &str) -> Option<(String, u64)> { + let self_cg = std::fs::read_to_string("/proc/self/cgroup").ok()?; + host_budget_tightest_cgroup_under(&self_cg, std::path::Path::new("/sys/fs/cgroup"), limit_file) +} + +pub fn host_budget_cgroup_v1_unlimited_bytes(page_size: u64) -> u64 { + (i64::MAX as u64 / page_size) * page_size +} + +pub fn host_budget_cgroup_v1() -> Option<(String, HostBudgetCgroupV1)> { + let self_cg = std::fs::read_to_string("/proc/self/cgroup").ok()?; + let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?; + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page_size <= 0 { + return Some(( + "/proc/self/mountinfo".to_string(), + HostBudgetCgroupV1::Unparseable("sysconf(_SC_PAGESIZE) unreadable".to_string()), + )); + } + host_budget_cgroup_v1_under( + std::path::Path::new("/"), + &self_cg, + &mountinfo, + page_size as u64, + ) +} + +pub fn host_budget_cgroup_v1_under( + fs_root: &std::path::Path, + self_cg: &str, + mountinfo: &str, + page_size: u64, +) -> Option<(String, HostBudgetCgroupV1)> { + let dir = host_budget_cgroup_v1_memory_dir(self_cg, mountinfo)?; + let dir_path = std::path::Path::new(&dir); + let joined = fs_root.join(dir_path.strip_prefix("/").unwrap_or(dir_path)); + let value = match std::fs::read_to_string(joined.join("memory.stat")) { + Ok(stat) => host_budget_cgroup_v1_from_stat(&stat, page_size), + Err(e) => HostBudgetCgroupV1::Unparseable(format!("memory.stat: {}", e)), + }; + Some((joined.display().to_string(), value)) +} + +pub fn host_budget_cgroup_v1_memory_dir(self_cg: &str, mountinfo: &str) -> Option { + let (mount_root, mount_point) = mountinfo.lines().find_map(|line| { + let fields: Vec<&str> = line.split(' ').collect(); + let dash = fields.iter().position(|f| *f == "-")?; + let fstype = fields.get(dash + 1)?; + let super_opts = fields.get(dash + 3)?; + if *fstype == "cgroup" && super_opts.split(',').any(|o| o == "memory") { + Some((fields.get(3)?.to_string(), fields.get(4)?.to_string())) + } else { + None + } + })?; + let path = self_cg.lines().find_map(|l| { + let mut parts = l.splitn(3, ':'); + let (_id, controllers, path) = (parts.next()?, parts.next()?, parts.next()?); + controllers + .split(',') + .any(|c| c == "memory") + .then(|| path.trim().to_string()) + })?; + let rel = if mount_root == "/" { + path.as_str() + } else { + let rest = path.strip_prefix(mount_root.as_str())?; + if !(rest.is_empty() || rest.starts_with('/')) { + return None; + } + rest + }; + Some( + std::path::Path::new(&mount_point) + .join(rel.trim_start_matches('/')) + .display() + .to_string(), + ) +} + +pub fn host_budget_cgroup_v1_from_stat(memory_stat: &str, page_size: u64) -> HostBudgetCgroupV1 { + let hits: std::vec::Vec<&str> = memory_stat + .lines() + .filter_map(|l| l.trim().strip_prefix("hierarchical_memory_limit ")) + .collect(); + let [body] = hits.as_slice() else { + return HostBudgetCgroupV1::Unparseable(memory_stat.to_string()); + }; + let unlimited = host_budget_cgroup_v1_unlimited_bytes(page_size); + match body.trim().parse::() { + Ok(n) if n < 0 => HostBudgetCgroupV1::Unparseable(body.to_string()), + Ok(n) if n >= unlimited as i128 => HostBudgetCgroupV1::Unlimited, + Ok(n) => HostBudgetCgroupV1::Limited(n as u64), + Err(_) => HostBudgetCgroupV1::Unparseable(body.to_string()), + } +} + +/// Darwin `hw.memsize` via `sysctlbyname`. Authority: `extdeps.darwin.sysctl` `HwMemsize`. +pub fn host_budget_darwin_physical() -> Option { + #[cfg(target_os = "macos")] + { + let name = std::ffi::CStr::from_bytes_with_nul(b"hw.memsize\0").ok()?; + let mut value: u64 = 0; + let mut len: libc::size_t = std::mem::size_of::() as libc::size_t; + let rc = unsafe { + libc::sysctlbyname( + name.as_ptr(), + (&mut value as *mut u64).cast::(), + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + if rc == 0 && value > 0 { + Some(value) + } else { + None + } + } + #[cfg(not(target_os = "macos"))] + { + None + } +}