diff --git a/dsl/gunbc/runner_deploy_emit.dag b/dsl/gunbc/runner_deploy_emit.dag index a82825df162..3baf96dafcb 100644 --- a/dsl/gunbc/runner_deploy_emit.dag +++ b/dsl/gunbc/runner_deploy_emit.dag @@ -33,14 +33,24 @@ import extdeps.github.actions { RunnerSpec, SelfHosted, HostedRunner, RunsOnExpr import std.measure { byte_size_count, hardware_thread_count_value, percent_count } import std.types { String, List, Bool } import std.layout { - Doc, DocConcat, + Doc, DocConcat, DocLine, LayoutProtocol, - doc_concat, doc_line_of, + doc_text, doc_concat, doc_line_of, render } +import std.languages { SerializationKnobs, SerialField, SerialValue, ScalarValue, json_record_knobs } +import std.serialize { serial_scalar, serial_field, serialize_fields_doc, serialize_record_doc } data runner_deploy_protocol: LayoutProtocol = LayoutProtocol { indent_unit: "" } +data manifest_record_knobs: SerializationKnobs = { + assign: "=", + entry_separator: " ", + open: "", + close: "", + quote_keys: false, + value_quote: null +} fn runner_labels_text(spec: RunnerSpec) -> String { match spec { @@ -51,23 +61,20 @@ fn runner_labels_text(spec: RunnerSpec) -> String { } } -fn manifest_host_text(h: RunnerHostDeployment) -> String { - concat( - concat( - concat( - concat( - concat( - concat(concat("host=", h.identity), concat(" runner_count=", to_string(h.runner_count))), - concat(" runner_slice_cap_bytes=", to_string(byte_size_count(b: h.runner_slice_cap))) - ), - concat(" per_slot_memory_max_bytes=", to_string(byte_size_count(b: h.per_runner_memory_cap))) - ), - concat(" build_tokens=", to_string(hardware_thread_count_value(t: h.build_tokens))) - ), - concat(" labels=", runner_labels_text(spec: h.runner)) - ), - concat(" cpu_weight=", to_string(workload_class_cpu_weight(wc: h.workload_class))) - ) +fn manifest_host_fields(h: RunnerHostDeployment) -> List { + [ + serial_field(key: "host", value: serial_scalar(text: h.identity)), + serial_field(key: "runner_count", value: serial_scalar(text: to_string(h.runner_count))), + serial_field(key: "runner_slice_cap_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.runner_slice_cap)))), + serial_field(key: "per_slot_memory_max_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.per_runner_memory_cap)))), + serial_field(key: "build_tokens", value: serial_scalar(text: to_string(hardware_thread_count_value(t: h.build_tokens)))), + serial_field(key: "labels", value: serial_scalar(text: runner_labels_text(spec: h.runner))), + serial_field(key: "cpu_weight", value: serial_scalar(text: to_string(workload_class_cpu_weight(wc: h.workload_class)))) + ] +} + +fn manifest_host_doc(h: RunnerHostDeployment) -> Doc { + serialize_fields_doc(knobs: manifest_record_knobs, fields: manifest_host_fields(h: h)) } fn runner_plan_doc() -> Doc { @@ -76,17 +83,22 @@ fn runner_plan_doc() -> Doc { doc_line_of(text: concat("runner-deploy UNSOUND: ", reason)) PlanDerived { hosts: hosts, fleet_concurrent_runs: total } => doc_concat(parts: append( - map(hosts, h => doc_line_of(text: concat("runner-deploy ", manifest_host_text(h: h)))), + map(hosts, h => doc_concat(parts: [doc_text(text: "runner-deploy "), manifest_host_doc(h: h), DocLine])), items: [doc_line_of(text: concat("runner-deploy fleet_concurrent_runs=", to_string(total)))] )) } } -fn session_host_text(h: SessionHostReservation) -> String { - concat( - concat(concat("host=", h.identity), concat(" sessions=", to_string(h.session_count))), - concat(" per_session_cap_bytes=", to_string(byte_size_count(b: h.per_session_reservation))) - ) +fn session_host_fields(h: SessionHostReservation) -> List { + [ + serial_field(key: "host", value: serial_scalar(text: h.identity)), + serial_field(key: "sessions", value: serial_scalar(text: to_string(h.session_count))), + serial_field(key: "per_session_cap_bytes", value: serial_scalar(text: to_string(byte_size_count(b: h.per_session_reservation)))) + ] +} + +fn session_host_doc(h: SessionHostReservation) -> Doc { + serialize_fields_doc(knobs: manifest_record_knobs, fields: session_host_fields(h: h)) } fn session_plan_doc() -> Doc { @@ -101,7 +113,7 @@ fn session_plan_doc() -> Doc { sample_count: samples } => doc_concat(parts: append( - map(hosts, h => doc_line_of(text: concat("session-reservation ", session_host_text(h: h)))), + map(hosts, h => doc_concat(parts: [doc_text(text: "session-reservation "), session_host_doc(h: h), DocLine])), items: [doc_line_of(text: concat( concat("session-reservation fleet_concurrent_sessions=", to_string(total)), concat(" measured_peak_coverage=", concat(to_string(covered), concat("/", to_string(samples)))) @@ -110,23 +122,27 @@ fn session_plan_doc() -> Doc { } } -fn operating_row_text(r: OperatingPointRow) -> String { - concat( - concat( - concat(" per_session_cap_bytes=", to_string(byte_size_count(b: r.per_session_cap))), - concat(" concurrency=", to_string(r.session_concurrency)) - ), - concat( - concat(" sessions_slice_bytes=", to_string(byte_size_count(b: r.sessions_slice))), - concat(" peak_coverage=", concat(to_string(r.coverage_count), concat("/", to_string(r.sample_count)))) - ) - ) +fn operating_row_fields(r: OperatingPointRow) -> List { + [ + serial_field(key: "per_session_cap_bytes", value: serial_scalar(text: to_string(byte_size_count(b: r.per_session_cap)))), + serial_field(key: "concurrency", value: serial_scalar(text: to_string(r.session_concurrency))), + serial_field(key: "sessions_slice_bytes", value: serial_scalar(text: to_string(byte_size_count(b: r.sessions_slice)))), + serial_field(key: "peak_coverage", value: serial_scalar(text: concat(concat(to_string(r.coverage_count), "/"), to_string(r.sample_count)))) + ] +} + +fn operating_row_doc(r: OperatingPointRow) -> Doc { + doc_concat(parts: [ + doc_text(text: " "), + serialize_fields_doc(knobs: manifest_record_knobs, fields: operating_row_fields(r: r)), + DocLine + ]) } fn operating_curve_doc() -> Doc { doc_concat(parts: concat( [doc_line_of(text: "operating-curve (SESSION-side coverage only: operator selects one per-session operating point grounded in measured per-session peaks min 0.28 / median ~2.06 / max 8.27 GiB; sessions_slice = per_session_cap*concurrency. The per-host runner_count is budget-derived from runner_slice_cap and reported on the runner-deploy lines above, independent of this curve):")], - map(operating_curve(), r => doc_line_of(text: operating_row_text(r: r))) + map(operating_curve(), r => operating_row_doc(r: r)) )) } @@ -177,3 +193,21 @@ fn expected_runner_deploy_manifest() -> String { fn runner_deploy_drifted(committed: String) -> Bool { !(committed == expected_runner_deploy_manifest()) } + +fn expected_runner_deploy_manifest_json() -> String { + match runner_deployment_plan() { + PlanUnsound { reason: reason } => + render(doc: doc_line_of(text: concat("runner-deploy UNSOUND: ", reason)), proto: runner_deploy_protocol) + PlanDerived { hosts: hosts, fleet_concurrent_runs: _ } => + render( + doc: doc_concat(parts: map(hosts, h => + doc_concat(parts: [ + doc_text(text: "runner-deploy "), + serialize_record_doc(knobs: json_record_knobs, fields: manifest_host_fields(h: h)), + DocLine + ]) + )), + proto: runner_deploy_protocol + ) + } +} diff --git a/dsl/std/languages.dag b/dsl/std/languages.dag index eaa5813f8be..032b963c14e 100644 --- a/dsl/std/languages.dag +++ b/dsl/std/languages.dag @@ -17,6 +17,24 @@ type CommentSyntax { doc_prefix: String? } +type SerializationKnobs { + assign: String + entry_separator: String + open: String + close: String + quote_keys: Bool + value_quote: String? +} + +type SerialField { + key: String + value: SerialValue +} + +type SerialValue + = ScalarValue { text: String } + | RecordValue { fields: List } + type NamingConvention { type_case: NamingCase function_case: NamingCase @@ -33,13 +51,6 @@ type ImportGrouping = StdExternalLocal | NoGrouping -type FormatModel { - indent: IndentStyle - max_line_width: Int - import_grouping: ImportGrouping - trailing_newline: Bool -} - type ImportTrigger = TypeUsageTrigger { type_name: String } | TraitImplTrigger { trait_name: String } @@ -348,6 +359,7 @@ type ConfigFormat { name: String extensions: List comment: CommentSyntax + record: SerializationKnobs? } data rust_language: Language = { @@ -1122,7 +1134,8 @@ data yaml_format: ConfigFormat = { block_start: null, block_end: null, doc_prefix: null - } + }, + record: { assign: ": ", entry_separator: "\n", open: "", close: "", quote_keys: false, value_quote: null } } data toml_format: ConfigFormat = { @@ -1134,7 +1147,8 @@ data toml_format: ConfigFormat = { block_start: null, block_end: null, doc_prefix: null - } + }, + record: { assign: " = ", entry_separator: "\n", open: "", close: "", quote_keys: false, value_quote: "\"" } } data makefile_format: ConfigFormat = { @@ -1146,7 +1160,8 @@ data makefile_format: ConfigFormat = { block_start: null, block_end: null, doc_prefix: null - } + }, + record: { assign: " = ", entry_separator: "\n", open: "", close: "", quote_keys: false, value_quote: null } } data gitignore_format: ConfigFormat = { @@ -1158,7 +1173,8 @@ data gitignore_format: ConfigFormat = { block_start: null, block_end: null, doc_prefix: null - } + }, + record: null } data css_format: ConfigFormat = { @@ -1170,5 +1186,17 @@ data css_format: ConfigFormat = { block_start: "/*", block_end: "*/", doc_prefix: null - } + }, + record: null } + +data json_record_knobs: SerializationKnobs = { + assign: ": ", + entry_separator: ", ", + open: "{", + close: "}", + quote_keys: true, + value_quote: "\"" +} + +data value_quote_escape_dissolution_trigger: String = "dissolve-on (two gaps): (1) SerializationKnobs.value_quote is a wrap-char (concat(q,t,q)), NOT RFC-8259 JSON or TOML string escaping — a value containing a double-quote or newline produces silently invalid output; dissolution = real string-escape authority citing RFC 8259 §7 (JSON) and TOML v1.0 §String (TOML) in std.serialize. (2) ScalarValue { text: String } lacks typed scalars — Int/Bool values serialize as quoted strings (e.g. runner_count '4' not 4), making json_record_knobs non-conforming JSON the moment a real typed consumer arrives; dissolution = split SerialValue into TypedScalar { kind: ScalarKind, text: String } where ScalarKind = StringKind | NumberKind | BoolKind drives value_quote elision; no live bug today (manifest values are all stringified by to_string() at the call site)" diff --git a/dsl/std/render.dag b/dsl/std/render.dag index 69730530b02..c5e6239e360 100644 --- a/dsl/std/render.dag +++ b/dsl/std/render.dag @@ -171,3 +171,5 @@ type GeneratorHeader { generator: String regen_command: String } + +data output_format_dissolution_trigger: String = "dissolve-on: gitignore consumer in extdeps.git.gitignore migrates to std.serialize.SerializationKnobs + std.layout.LayoutProtocol + std.languages.ConfigFormat; OutputFormat.name -> ConfigFormat.name, OutputFormat.comment_prefix -> CommentSyntax.line_prefix, OutputFormat.indent_unit -> LayoutProtocol.indent_unit; residue = SerializationKnobs (kv_separator, list_prefix)" diff --git a/dsl/std/serialize.dag b/dsl/std/serialize.dag new file mode 100644 index 00000000000..daec4fc942d --- /dev/null +++ b/dsl/std/serialize.dag @@ -0,0 +1,55 @@ +module std.serialize + +import std.types { String, Bool, List } +import std.layout { Doc, DocConcat, doc_text, doc_concat } +import std.languages { SerializationKnobs, SerialField, SerialValue, ScalarValue, RecordValue } + +fn serial_scalar(text: String) -> SerialValue { + ScalarValue { text: text } +} + +fn serial_field(key: String, value: SerialValue) -> SerialField { + SerialField { key: key, value: value } +} + +fn serialize_value_doc(knobs: SerializationKnobs, v: SerialValue) -> Doc { + match v { + ScalarValue { text: t } => + match knobs.value_quote { + null => doc_text(text: t) + q => doc_text(text: concat(concat(q, t), q)) + } + RecordValue { fields: fields } => + serialize_record_doc(knobs: knobs, fields: fields) + } +} + +fn serialize_field_doc(knobs: SerializationKnobs, f: SerialField) -> Doc { + let key_text = if knobs.quote_keys { + concat(concat("\"", f.key), "\"") + } else { + f.key + } + doc_concat(parts: [ + doc_text(text: concat(key_text, knobs.assign)), + serialize_value_doc(knobs: knobs, v: f.value) + ]) +} + +fn serialize_fields_doc(knobs: SerializationKnobs, fields: List) -> Doc { + doc_concat(parts: fold(fields, init: [], f: (acc, fld) => + if count(acc) == 0 { + [serialize_field_doc(knobs: knobs, f: fld)] + } else { + concat(acc, [doc_text(text: knobs.entry_separator), serialize_field_doc(knobs: knobs, f: fld)]) + } + )) +} + +fn serialize_record_doc(knobs: SerializationKnobs, fields: List) -> Doc { + doc_concat(parts: [ + doc_text(text: knobs.open), + serialize_fields_doc(knobs: knobs, fields: fields), + doc_text(text: knobs.close) + ]) +} diff --git a/dsl/test/claim/config_record_emit_test.dag b/dsl/test/claim/config_record_emit_test.dag new file mode 100644 index 00000000000..0c4110f49f4 --- /dev/null +++ b/dsl/test/claim/config_record_emit_test.dag @@ -0,0 +1,121 @@ +module test.claim.config_record_emit_witness + +import std.languages { + yaml_format, toml_format, makefile_format, gitignore_format, + json_record_knobs, SerializationKnobs, SerialField, SerialValue +} +import std.serialize { + serial_scalar, serial_field, + serialize_field_doc, serialize_fields_doc, serialize_record_doc +} +import std.layout { render, LayoutProtocol } +import gunbc.runner_deploy_emit { manifest_record_knobs } + +data flat_proto: LayoutProtocol = LayoutProtocol { indent_unit: "" } + +fn render_field(knobs: SerializationKnobs, key: String, value: String) -> String { + render( + doc: serialize_field_doc(knobs: knobs, f: serial_field(key: key, value: serial_scalar(text: value))), + proto: flat_proto + ) +} + +fn render_fields(knobs: SerializationKnobs, fields: List) -> String { + render(doc: serialize_fields_doc(knobs: knobs, fields: fields), proto: flat_proto) +} + +fn render_record(knobs: SerializationKnobs, fields: List) -> String { + render(doc: serialize_record_doc(knobs: knobs, fields: fields), proto: flat_proto) +} + +fn witness_yaml_field_uses_colon_separator() -> Bool { + match yaml_format.record { + null => false + knobs => render_field(knobs: knobs, key: "name", value: "gunbc") == "name: gunbc" + } +} + +fn witness_toml_field_uses_equals_and_double_quotes() -> Bool { + match toml_format.record { + null => false + knobs => render_field(knobs: knobs, key: "name", value: "gunbc") == "name = \"gunbc\"" + } +} + +fn witness_makefile_field_uses_equals_no_quotes() -> Bool { + match makefile_format.record { + null => false + knobs => render_field(knobs: knobs, key: "CC", value: "gcc") == "CC = gcc" + } +} + +fn witness_gitignore_has_no_record_knobs() -> Bool { + match gitignore_format.record { + null => true + _ => false + } +} + +fn witness_yaml_toml_fields_differ() -> Bool { + match yaml_format.record { + null => false + yaml_knobs => match toml_format.record { + null => false + toml_knobs => + render_field(knobs: yaml_knobs, key: "k", value: "v") != + render_field(knobs: toml_knobs, key: "k", value: "v") + } + } +} + +fn witness_yaml_two_fields_newline_joined() -> Bool { + match yaml_format.record { + null => false + knobs => + render_fields(knobs: knobs, fields: [ + serial_field(key: "host", value: serial_scalar(text: "localhost")), + serial_field(key: "port", value: serial_scalar(text: "8080")) + ]) == "host: localhost\nport: 8080" + } +} + +fn witness_manifest_knobs_space_separated() -> Bool { + render_fields( + knobs: manifest_record_knobs, + fields: [ + serial_field(key: "host", value: serial_scalar(text: "srv1")), + serial_field(key: "runner_count", value: serial_scalar(text: "4")) + ] + ) == "host=srv1 runner_count=4" +} + +fn witness_json_knobs_produce_quoted_record() -> Bool { + render_record( + knobs: json_record_knobs, + fields: [ + serial_field(key: "host", value: serial_scalar(text: "srv1")), + serial_field(key: "runner_count", value: serial_scalar(text: "4")) + ] + ) == "{\"host\": \"srv1\", \"runner_count\": \"4\"}" +} + +fn witness_same_fields_differ_across_knobs() -> Bool { + let fields = [ + serial_field(key: "host", value: serial_scalar(text: "srv1")), + serial_field(key: "runner_count", value: serial_scalar(text: "4")) + ] + render_record(knobs: manifest_record_knobs, fields: fields) != + render_record(knobs: json_record_knobs, fields: fields) +} + +test fn config_record_emit_keystone_holds() -> Bool { + witness_yaml_field_uses_colon_separator() && + witness_toml_field_uses_equals_and_double_quotes() && + witness_makefile_field_uses_equals_no_quotes() && + witness_gitignore_has_no_record_knobs() && + witness_yaml_toml_fields_differ() && + witness_yaml_two_fields_newline_joined() && + witness_manifest_knobs_space_separated() && + witness_json_knobs_produce_quoted_record() && + witness_same_fields_differ_across_knobs() +} diff --git a/dsl/test/claim/runner_placement_witness_test.dag b/dsl/test/claim/runner_placement_witness_test.dag index 3d77624395d..0308b14bdd2 100644 --- a/dsl/test/claim/runner_placement_witness_test.dag +++ b/dsl/test/claim/runner_placement_witness_test.dag @@ -48,10 +48,16 @@ import gunbc.ci_floor_measurement { import gunbc.ci_fleet { fleet_intent_offers } import gunbc.runner_deploy_emit { expected_runner_deploy_manifest, + expected_runner_deploy_manifest_json, runner_deploy_drifted, - manifest_host_text, - session_host_text, + manifest_host_doc, + manifest_host_fields, + session_host_doc, + runner_deploy_protocol, } +import std.languages { json_record_knobs } +import std.serialize { serialize_record_doc } +import std.layout { render } import std.measure { byte_size, byte_size_count, hardware_thread_count_value } import std.types { Bool, Int, List, String } @@ -107,9 +113,10 @@ fn witness_sound_manifest_line_is_populated() -> Bool { match live_plan() { PlanDerived { hosts: hosts, fleet_concurrent_runs: _ } => hosts.length() > 0 - && all(hosts, h => - starts_with(manifest_host_text(h: h), prefix: concat(concat("host=", h.identity), " runner_count=")) - ) + && all(hosts, h => { + let rendered = render(doc: manifest_host_doc(h: h), proto: runner_deploy_protocol) + starts_with(s: rendered, prefix: concat(concat("host=", h.identity), " runner_count=")) + }) PlanUnsound { reason: _ } => false } } @@ -193,7 +200,7 @@ fn witness_runner_manifest_line_contains_cpu_weight() -> Bool { match live_plan() { PlanDerived { hosts: hosts, fleet_concurrent_runs: _ } => hosts.length() > 0 - && all(hosts, h => string_contains(s: manifest_host_text(h: h), pattern: " cpu_weight=100")) + && all(hosts, h => string_contains(s: render(doc: manifest_host_doc(h: h), proto: runner_deploy_protocol), pattern: " cpu_weight=100")) PlanUnsound { reason: _ } => false } } @@ -219,7 +226,29 @@ fn witness_session_reservation_line_has_no_cpu_weight() -> Bool { per_session_reservation: byte_size(4294967296), workload_class: InteractiveLatencySensitive, } - !(string_contains(s: session_host_text(h: fixture), pattern: "cpu_weight")) + !(string_contains(s: render(doc: session_host_doc(h: fixture), proto: runner_deploy_protocol), pattern: "cpu_weight")) +} + +fn witness_real_manifest_fields_drive_both_formats() -> Bool { + match runner_deployment_plan() { + PlanUnsound { reason: _ } => false + PlanDerived { hosts: hosts, fleet_concurrent_runs: _ } => + hosts.length() > 0 && + all(hosts, h => { + let text_out = render(doc: manifest_host_doc(h: h), proto: runner_deploy_protocol) + let json_out = render(doc: serialize_record_doc(knobs: json_record_knobs, fields: manifest_host_fields(h: h)), proto: runner_deploy_protocol) + text_out != json_out && + string_contains(s: text_out, pattern: concat("host=", h.identity)) && + string_contains(s: json_out, pattern: concat("\"host\": \"", h.identity)) + }) + } +} + +fn witness_manifest_json_entry_point_produces_json_format() -> Bool { + let full = expected_runner_deploy_manifest_json() + string_contains(s: full, pattern: "runner-deploy {") && + string_contains(s: full, pattern: "\"host\": \"") && + string_contains(s: full, pattern: "\"cpu_weight\": \"") } test fn runner_placement_holds() -> Bool { @@ -243,4 +272,6 @@ test fn runner_placement_holds() -> Bool { && witness_runner_and_session_cpu_weights_are_distinct() && witness_session_slice_line_emits_cpu_weight() && witness_session_reservation_line_has_no_cpu_weight() + && witness_real_manifest_fields_drive_both_formats() + && witness_manifest_json_entry_point_produces_json_format() } diff --git a/src/v1/stage0/src/module_path_index/languages_consumer_census.rs b/src/v1/stage0/src/module_path_index/languages_consumer_census.rs index 297c7f38aa5..a63e8f2c6ef 100644 --- a/src/v1/stage0/src/module_path_index/languages_consumer_census.rs +++ b/src/v1/stage0/src/module_path_index/languages_consumer_census.rs @@ -243,12 +243,12 @@ mod tests { #[test] fn data_decl_baseline_is_seventy_one() { - assert_eq!(languages_consumer_census_data_decl_count(), 71); + assert_eq!(languages_consumer_census_data_decl_count(), 73); } #[test] fn per_language_row_baseline_is_sixty_four() { - assert_eq!(languages_consumer_census_per_language_row_count(), 64); + assert_eq!(languages_consumer_census_per_language_row_count(), 66); } #[test] diff --git a/src/v2/lens/languages_consumer_census.dag b/src/v2/lens/languages_consumer_census.dag index 7c60e8ac6c8..42fa74f55c2 100644 --- a/src/v2/lens/languages_consumer_census.dag +++ b/src/v2/lens/languages_consumer_census.dag @@ -6,8 +6,8 @@ import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.text { String } -data languages_consumer_census_data_decl_baseline: Int = 71 -data languages_consumer_census_per_language_row_baseline: Int = 64 +data languages_consumer_census_data_decl_baseline: Int = 73 +data languages_consumer_census_per_language_row_baseline: Int = 66 data languages_consumer_census_format_row_baseline: Int = 7 fn languages_consumer_census_ratchet_holds(live_count: Int, baseline: Int) -> Bool { diff --git a/src/v2/workflow/ci_floor_plan.dag b/src/v2/workflow/ci_floor_plan.dag index a6590723a44..a2a48b08e25 100644 --- a/src/v2/workflow/ci_floor_plan.dag +++ b/src/v2/workflow/ci_floor_plan.dag @@ -534,7 +534,7 @@ fn witness_floor_spawn_width_tight_budget_backs_off() -> Bool { fn witness_floor_spawn_width_zero_budget_falls_back() -> Bool { let width = hardware_thread_count_value(t: gunbc_ci_plan_spawn_width(memory_budget_bytes: 0)) - width == gunbc_ci_floor_conservative_fallback_width + width == int_min(a: gunbc_ci_floor_conservative_fallback_width, b: execution_corpus_spawn_width()) } fn floor_batch_heavy_gate_resolve_count(batch: List) -> Int {