diff --git a/core/daglang/daglang-lower/src/lib.rs b/core/daglang/daglang-lower/src/lib.rs index 001c636ad18..7af7994842c 100644 --- a/core/daglang/daglang-lower/src/lib.rs +++ b/core/daglang/daglang-lower/src/lib.rs @@ -5678,13 +5678,16 @@ fn add_service_call_edges( known_interface_types: &HashSet, data_values: &HashMap, ) -> Result<(), LowerError> { + // Track transport endpoint usage across ALL callables in ALL modules so + // that the second callable to reference the same service operation gets + // a cloned triplet (_c1, _c2, …) instead of wiring duplicate scalar + // edges to the original. Previously this was per-module, which meant + // two modules calling the same operation both wired to the shared + // prepare node — causing "multiple upstream edges" at execution time + // (BT-E1). + let mut endpoint_use_count: HashMap = HashMap::new(); for module in &project.modules { let module_name = module.module_path.as_dotted(); - // Track transport endpoint usage across ALL callables in the module so - // that the second callable to reference the same service operation gets - // a cloned triplet (_c1, _c2, …) instead of wiring duplicate scalar - // edges to the original. - let mut endpoint_use_count: HashMap = HashMap::new(); for item in &module.ast.items { let (item_name, params, stmts, uses_binding_types, body_lossy) = match &item.node { Item::FnDef(def) => ( diff --git a/core/daglang/daglang-lower/src/tests.rs b/core/daglang/daglang-lower/src/tests.rs index 348508e8419..c2a196d9da5 100644 --- a/core/daglang/daglang-lower/src/tests.rs +++ b/core/daglang/daglang-lower/src/tests.rs @@ -3273,3 +3273,73 @@ func caller(id: String) -> { name: String } { "error should mention service, operation, and missing transport; got: {msg}", ); } + +/// BT-E1: Two modules calling the same service operation must each get their +/// own transport triplet clone. Before the fix, `endpoint_use_count` was +/// per-module, so the second module would wire to the original shared prepare +/// node — producing duplicate scalar edges that fail at execution time. +#[test] +fn cross_module_service_call_gets_cloned_triplet() { + let typed = typed_project_from_sources(&[ + ( + "dsl/services/api.dag", + r#"module shared.api +service remote.Api { + operation Fetch(query: String) -> { data: String } { + transport rest { method: GET, path: "/fetch" } + } +}"#, + ), + ( + "dsl/callers/alpha.dag", + r#"module callers.alpha +import shared.api +func alpha_fetch(q: String) -> { data: String } { + result = remote.Api.Fetch(query: q) + return { data: result.data } +}"#, + ), + ( + "dsl/callers/beta.dag", + r#"module callers.beta +import shared.api +func beta_fetch(q: String) -> { data: String } { + result = remote.Api.Fetch(query: q) + return { data: result.data } +}"#, + ), + ]); + let dag = lower_typed_project(&typed).expect("lowering should succeed (BT-E1)"); + + // The original triplet exists. + let suffix = "shared_api_remote_Api_Fetch"; + let original_prepare = format!("prepare_transport_{suffix}"); + assert!( + dag.nodes.iter().any(|n| n.id.0 == original_prepare), + "original prepare node should exist", + ); + + // The second caller should get a cloned triplet (_c1 suffix). + let cloned_prepare = format!("prepare_transport_{suffix}_c1"); + assert!( + dag.nodes.iter().any(|n| n.id.0 == cloned_prepare), + "cloned prepare node should exist for second caller (BT-E1): nodes = {:?}", + dag.nodes.iter().map(|n| &n.id.0).collect::>(), + ); + + // Both prepare nodes should have their own scalar inputs (no shared wiring). + let original_edges: Vec<_> = dag + .edges + .iter() + .filter(|e| e.to_node.0 == original_prepare) + .collect(); + let cloned_edges: Vec<_> = dag + .edges + .iter() + .filter(|e| e.to_node.0 == cloned_prepare) + .collect(); + assert!( + !original_edges.is_empty() && !cloned_edges.is_empty(), + "both triplets should have incoming edges", + ); +} diff --git a/core/daglang/daglang-resolve/tests/module_graph.rs b/core/daglang/daglang-resolve/tests/module_graph.rs index fee15613d64..69cec4f1f35 100644 --- a/core/daglang/daglang-resolve/tests/module_graph.rs +++ b/core/daglang/daglang-resolve/tests/module_graph.rs @@ -225,6 +225,8 @@ fn real_corpus_dependency_counts_match_expected_snapshot() { ("shared.gist_modes".into(), 5), ("std.access".into(), 0), ("std.box_draw".into(), 3), + ("std.ci".into(), 0), + ("std.ci_render".into(), 1), ("std.fermi".into(), 1), ("std.fidelity".into(), 2), ("std.filesystem".into(), 1), @@ -243,6 +245,7 @@ fn real_corpus_dependency_counts_match_expected_snapshot() { ("std.width".into(), 2), ("tools.bootstrap".into(), 4), ("tools.build".into(), 3), + ("tools.cigen".into(), 3), ("tools.clippy".into(), 4), ("tools.codegen".into(), 2), ("tools.deps".into(), 4), diff --git a/core/daglang/daglang-syntax/tests/common/mod.rs b/core/daglang/daglang-syntax/tests/common/mod.rs index 213caf832ed..b8733b881d9 100644 --- a/core/daglang/daglang-syntax/tests/common/mod.rs +++ b/core/daglang/daglang-syntax/tests/common/mod.rs @@ -107,6 +107,8 @@ pub fn expected_dsl_files_sorted() -> Vec<&'static str> { "shared/gist_modes.dag", "std/access.dag", "std/box_draw.dag", + "std/ci.dag", + "std/ci_render.dag", "std/fermi.dag", "std/fidelity.dag", "std/filesystem.dag", @@ -125,6 +127,7 @@ pub fn expected_dsl_files_sorted() -> Vec<&'static str> { "std/width.dag", "tools/bootstrap.dag", "tools/build.dag", + "tools/cigen.dag", "tools/clippy.dag", "tools/codegen.dag", "tools/deps.dag", diff --git a/dsl/std/ci.dag b/dsl/std/ci.dag new file mode 100644 index 00000000000..945fdb0295b --- /dev/null +++ b/dsl/std/ci.dag @@ -0,0 +1,150 @@ +// std/ci.dag -- CI model types: tautological definitions. +// +// "What is a CI workflow?" -- provider-independent model of CI/CD pipelines. +// Each type is a compositional building block. Data declarations are +// tautological assertions about shared CI configurations (Rust cache paths, +// cargo env, default runners). +// +// Layering: +// Layer 0: Primitives (CiStepKind, CiTriggerEvent, CiPermissionLevel) +// Layer 1: Building blocks (CiStep, CiTrigger, CiPermission, CiCache, CiEnv) +// Layer 2: Composition (CiJob, CiWorkflow) +// Layer 3: Provider (CiProvider = GitHub | GitLab) +// +// Consumers: std/ci_render.dag (rendering), tools/cigen.dag (generation). + +module std.ci + +// ── Layer 0: Primitives ────────────────────────────────────────────── + +// "What kind of step?" -- the three shapes a CI step can take. +type CiStepKind + = Run { command: String } + | Uses { action: String, with: List } + | DagRun { binary: String, args: List } + +// "What triggers a workflow?" +type CiTriggerEvent + = Push + | PullRequest + | Schedule { cron: String } + | WorkflowDispatch + +// "What permission level?" +type CiPermissionLevel + = PermRead + | PermWrite + | PermNone + +// "Which CI provider?" +type CiProvider + = GitHub + | GitLab + +// ── Layer 1: Building blocks ───────────────────────────────────────── + +// "What is an environment variable?" +type CiEnv { + key: String + value: String +} + +// "What is a permission scope?" +type CiPermission { + scope: String + level: CiPermissionLevel +} + +// "What is a CI step?" +type CiStep { + name: String + kind: CiStepKind + env: List +} + +// "What is a trigger?" +type CiTrigger { + event: CiTriggerEvent + branches: List +} + +// "What is a cache?" +type CiCache { + key: String + paths: List + restore_keys: List +} + +// "What is a checkout?" +type CiCheckout { + fetch_depth: Int? + submodules: String? +} + +// ── Layer 2: Composition ───────────────────────────────────────────── + +// "What is a CI job?" +type CiJob { + name: String + runner: String + timeout_minutes: Int + steps: List +} + +// "What is a CI workflow?" +type CiWorkflow { + name: String + triggers: List + permissions: List + env: List + jobs: List + cache: CiCache? + checkout: CiCheckout? + secrets: List +} + +// "What is a CI config?" -- discovery result from Rust extern bridge. +type CiConfig { + workflow_name: String + runner: String + timeout_minutes: Int + branches: List + permissions: List + env: List + cache: CiCache + checkout: CiCheckout + secrets: List + tool_command: String + generator_name: String + regenerate_command: String +} + +// ── Layer 3: Shared data declarations ──────────────────────────────── + +// "What is the Rust cargo cache?" +data rust_cache: CiCache = { + key: "cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}", + paths: [ + "~/.cargo/bin/", + "~/.cargo/registry/index/", + "~/.cargo/registry/cache/", + "~/.cargo/git/db/" + ], + restore_keys: ["cargo-${{ runner.os }}-"] +} + +// "What is the standard Rust CI env?" +data rust_ci_env: List = [ + { key: "CARGO_TERM_COLOR", value: "always" }, + { key: "RUSTFLAGS", value: "-D warnings" } +] + +// "What is the default checkout?" +data default_checkout: CiCheckout = { + fetch_depth: null, + submodules: null +} + +// Output paths for provider-specific YAML. +data github_output_path: String = ".github/workflows/ci.yml" +data gitlab_output_path: String = ".gitlab-ci.yml" diff --git a/dsl/std/ci_render.dag b/dsl/std/ci_render.dag new file mode 100644 index 00000000000..c117a19461e --- /dev/null +++ b/dsl/std/ci_render.dag @@ -0,0 +1,148 @@ +// std/ci_render.dag -- CI YAML rendering functions. +// +// Pure functions that render CiConfig into provider-specific YAML strings. +// No I/O, no extern calls. Follows the makegen.dag rendering pattern: +// small composable fns, |> map + |> join("\n"), YAML indentation via +// string literals (no general YAML serializer). +// +// Consumers: tools/cigen.dag (entry point). + +module std.ci_render + +import std.ci { + CiConfig, CiEnv, CiPermission, CiPermissionLevel, CiCache, CiCheckout +} + +// ── Shared helpers ─────────────────────────────────────────────────── + +fn render_ci_header(generator_name: String, regenerate_command: String) -> String { + "# Generated by {generator_name}\n# DO NOT EDIT - regenerate with: {regenerate_command}" +} + +fn render_permission_level(level: CiPermissionLevel) -> String { + match level { + PermRead => "read" + PermWrite => "write" + PermNone => "none" + } +} + +fn render_yaml_list_indented(indent: String, items: List) -> String { + items |> map(item => "{indent}- {item}") |> join("\n") +} + +fn render_env_block(indent: String, env: List) -> String { + if env |> count() == 0 { + "" + } else { + let lines = env |> map(e => "{indent}{e.key}: {e.value}") |> join("\n") + lines + } +} + +fn render_env_block_quoted(indent: String, env: List) -> String { + if env |> count() == 0 { + "" + } else { + let lines = env |> map(e => "{indent}{e.key}: \"{e.value}\"") |> join("\n") + lines + } +} + +// ── GitHub Actions rendering ───────────────────────────────────────── + +fn render_github_triggers(branches: List) -> String { + let branch_lines = branches |> map(b => " - {b}") |> join("\n") + "on:\n push:\n branches:\n{branch_lines}\n pull_request:\n branches:\n{branch_lines}" +} + +fn render_github_permissions(permissions: List) -> String { + if permissions |> count() == 0 { + "" + } else { + let lines = permissions |> map(p => " {p.scope}: {render_permission_level(level: p.level)}") |> join("\n") + "permissions:\n{lines}" + } +} + +fn render_github_env(env: List) -> String { + if env |> count() == 0 { + "" + } else { + let lines = render_env_block(indent: " ", env: env) + "env:\n{lines}" + } +} + +fn render_github_cache(cache: CiCache) -> String { + let paths = cache.paths |> map(p => " {p}") |> join("\n") + let restore = cache.restore_keys |> map(k => " {k}") |> join("\n") + " - name: Cache Cargo\n uses: actions/cache@v4\n with:\n path: |\n{paths}\n key: {cache.key}\n restore-keys: |\n{restore}" +} + +fn render_github_checkout(checkout: CiCheckout) -> String { + let base = " - name: Checkout\n uses: actions/checkout@v4" + match checkout.fetch_depth { + null => base + depth => "{base}\n with:\n fetch-depth: {depth}" + } +} + +fn render_github_secrets_env(secrets: List) -> String { + secrets |> map(s => " {s}: ${{{{ secrets.{s} }}}}") |> join("\n") +} + +fn render_github_run_step_env(secrets: List) -> String { + let incremental = " CARGO_INCREMENTAL: \"1\"" + if secrets |> count() == 0 { + " env:\n{incremental}" + } else { + let secret_lines = render_github_secrets_env(secrets: secrets) + " env:\n{incremental}\n{secret_lines}" + } +} + +fn render_github_bootstrap_step() -> String { + " - name: Verify Bootstrap Invariants\n run: |\n rm -rf target/codegen\n # Cargo validates all [[bin]] paths even with --bin filter.\n # Create minimal stubs so the manifest parses, then check only bootstrap binaries.\n for dir in $(grep 'path = \"../target/codegen/' gunbc-dag/Cargo.toml | sed 's|.*\"../\\(.*\\)/main.rs\"|\\1|'); do\n mkdir -p \"$dir\" && echo 'fn main() {}' > \"$dir/main.rs\"\n done\n cargo check -p gunbc-dag --bin gunbc-codegen --bin gunbc-ci" +} + +fn render_github_workflow(config: CiConfig) -> String { + let header = render_ci_header(generator_name: config.generator_name, regenerate_command: config.regenerate_command) + let name = "name: {config.workflow_name}" + let triggers = render_github_triggers(branches: config.branches) + let permissions = render_github_permissions(permissions: config.permissions) + let env = render_github_env(env: config.env) + let job_header = "jobs:\n {config.workflow_name}:\n runs-on: {config.runner}\n timeout-minutes: {config.timeout_minutes}\n steps:" + let checkout = render_github_checkout(checkout: config.checkout) + let setup_rust = " - name: Setup Rust\n uses: dtolnay/rust-toolchain@stable" + let cache = render_github_cache(cache: config.cache) + let bootstrap = render_github_bootstrap_step() + let run_step = " - name: Run CI Pipeline\n run: {config.tool_command}" + let run_env = render_github_run_step_env(secrets: config.secrets) + "{header}\n\n{name}\n\n{triggers}\n\n{permissions}\n\n{env}\n\n{job_header}\n{checkout}\n\n{setup_rust}\n\n{cache}\n\n{bootstrap}\n\n{run_step}\n{run_env}\n" +} + +// ── GitLab CI rendering ────────────────────────────────────────────── + +fn render_gitlab_variables(env: List) -> String { + if env |> count() == 0 { + "" + } else { + let lines = render_env_block_quoted(indent: " ", env: env) + "variables:\n{lines}" + } +} + +fn render_gitlab_cache() -> String { + "cache:\n key: cargo-${{CI_COMMIT_REF_SLUG}}\n paths:\n - .cargo/\n - target/" +} + +fn render_gitlab_ci(config: CiConfig) -> String { + let header = render_ci_header(generator_name: config.generator_name, regenerate_command: config.regenerate_command) + let image = "image: rust:latest" + let variables = render_gitlab_variables(env: config.env) + let stages = "stages:\n - ci" + let cache = render_gitlab_cache() + let job = "{config.workflow_name}:\n stage: ci\n script:\n - {config.tool_command}" + "{header}\n\n{image}\n\n{variables}\n\n{stages}\n\n{cache}\n\n{job}\n" +} diff --git a/dsl/tools/cigen.dag b/dsl/tools/cigen.dag new file mode 100644 index 00000000000..0bc8606140c --- /dev/null +++ b/dsl/tools/cigen.dag @@ -0,0 +1,35 @@ +// tools/cigen.dag -- CI YAML generation. +// +// Renders CI workflow files for GitHub Actions and GitLab CI from a +// single CiConfig discovery result. Follows the makegen.dag pattern: +// discover via extern → render in pure DSL → content_upsert. +// +// The only extern func is discover_ci_config(), which calls into Rust +// to introspect permissions, secrets, tool invocations, and branches. + +module tools.cigen + +import std.patterns { content_upsert } +import std.ci { CiConfig, github_output_path, gitlab_output_path } +import std.ci_render { render_github_workflow, render_gitlab_ci } + +// ── Extern boundary ────────────────────────────────────────────────── +// CI config discovery: permissions, secrets, env, runner, tool command. +// Rust builds CiConfig from ci_workflow_permissions(), ci_live_test_secrets(), +// CargoEnv::ci(), etc. + +extern func discover_ci_config() -> CiConfig + +// ── Entry point ────────────────────────────────────────────────────── + +func cigen() -> { github_written: Bool, gitlab_written: Bool } { + config = discover_ci_config() + + github_yaml = render_github_workflow(config: config) + github = content_upsert(content: github_yaml, path: github_output_path) + + gitlab_yaml = render_gitlab_ci(config: config) + gitlab = content_upsert(content: gitlab_yaml, path: gitlab_output_path) + + return { github_written: github.written, gitlab_written: gitlab.written } +} diff --git a/gunbc-dag/src/bin/codegen_cli.rs b/gunbc-dag/src/bin/codegen_cli.rs index b0592386820..26b9459bd07 100644 --- a/gunbc-dag/src/bin/codegen_cli.rs +++ b/gunbc-dag/src/bin/codegen_cli.rs @@ -24,21 +24,16 @@ use cargo_metadata::MetadataCommand; use gunbc_cli::BinaryArgs; use gunbc_codegen::{core_outputs, generate_cli_with_import, FileWriter, ToolDef}; -use gunbc_dag::WorkspaceBinary; use gunbc_exec::{print_attention, run_freshness_steps, AttentionLevel}; use gunbc_ir::resource::{ check_manifest_freshness, codegen_resource_def, load_manifest_default, update_resource_manifest, FreshnessOptions, ManagedResource, ManifestEntry, ManifestFreshness, ManifestUpdateError, ResourceDef, ResourceError, ResourceIo, ResourceManifest, }; -use gunbc_ir::transport::ci::{ - yaml_block, CacheConfig, CiRenderer, GitHubActionsProvider, GitLabCiProvider, RenderConfig, -}; use gunbc_ir::WorkspaceLayout; use gunbc_lib_transport::TransportIo; use std::collections::{HashMap, HashSet}; use std::env; -use std::fmt::Write; use std::path::{Path, PathBuf}; use toml_edit::{value, ArrayOfTables, DocumentMut, Item, Table}; @@ -348,264 +343,76 @@ fn cmd_codegen(dry_run: bool) { update_manifest_after_codegen(dry_run, &io); } -/// Generate CI workflow YAML files. +/// Generate CI workflow YAML files via DSL tool (CG-4). /// -/// Generates both GitHub Actions and GitLab CI configurations. +/// Builds and executes the `tools/cigen.dag` DSL graph, which renders +/// GitHub Actions and GitLab CI YAML through pure DSL rendering functions. +/// The only Rust-side work is `discover_ci_config()` (extern func). fn cmd_cigen(dry_run: bool) { - println!("gunbc-codegen: cigen"); + println!("gunbc-codegen: cigen (DSL)"); println!(" mode: {}", if dry_run { "dry-run" } else { "real" }); println!(); - let io = TransportIo::new(); - let writer = FileWriter::new(dry_run, &io); - - let github_provider = GitHubActionsProvider; - let gitlab_provider = GitLabCiProvider::default(); - - // Generate CI YAML for gunbc-ci - let codegen = WorkspaceBinary::Codegen.invocation(); - let tool = WorkspaceBinary::Ci.invocation(); - - // Derive permissions from CI workflow integrations (checkout, GCP WIF, etc.) - let ci_perms: Vec<(String, String)> = gunbc_dag::ci::ci_workflow_permissions() - .into_iter() - .map(|(scope, level)| { - ( - scope.as_yaml_key().to_string(), - level.as_yaml_value().to_string(), - ) - }) - .collect(); - - // Secrets required by live flow tests (derived from testgen metadata). - let ci_secrets: Vec = gunbc_dag::ci::ci_live_test_secrets() - .into_iter() - .map(|s| s.to_string()) - .collect(); - - let config = RenderConfig::new("ci", tool) - .with_generator(&codegen.binary, &format!("{} -- cigen", codegen.command())) - .with_runner(gunbc_ir::transport::github_actions::ubuntu_latest()) - .with_cargo_env(gunbc_ir::CargoEnv::ci()) - .with_git(gunbc_ir::GitConfig::default()) - .with_cache(CacheConfig::rust()) - .with_permissions(ci_perms) - .with_secrets_env(ci_secrets); - - let outputs: Vec<(&str, CiTemplateKind, String, String)> = vec![ - ( - "GitHub Actions", - CiTemplateKind::GitHubActions, - generate_github_actions_template(&config), - github_provider.output_path("ci"), - ), - ( - "GitLab CI", - CiTemplateKind::GitLabCi, - generate_gitlab_ci_template(&config), - gitlab_provider.output_path("ci"), - ), - ]; - - let mut had_errors = false; - for (label, kind, yaml, path) in &outputs { - if let Err(error) = validate_generated_ci_template(*kind, yaml) { - eprintln!(" [ci] {} validation ERROR: {}", label, error); - had_errors = true; - continue; - } - - match writer.write_if_changed(Path::new(path), yaml) { - Ok(result) => { - let status = if dry_run { - "dry-run" - } else if result.changed { - "written" - } else { - "unchanged" - }; - println!(" [ci] {} ({})", path, status); - } - Err(e) => { - eprintln!(" [ci] {} ERROR: {}", label, e); - had_errors = true; - } + let dag = match gunbc_dag::dsl_builder::build_dsl_graph_for_entrypoint( + "tools/cigen.dag", + Some("cigen"), + ) { + Ok(dag) => dag, + Err(e) => { + print_attention( + AttentionLevel::Error, + "DSL compilation failed", + &e.to_string(), + ); + std::process::exit(1); } - } - - if had_errors { - std::process::exit(1); - } - - println!(); - println!("Generated: {} CI files", outputs.len()); -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CiTemplateKind { - GitHubActions, - GitLabCi, -} + }; -fn validate_generated_ci_template(kind: CiTemplateKind, yaml: &str) -> Result<(), String> { - match kind { - CiTemplateKind::GitHubActions => validate_github_actions_template(yaml), - CiTemplateKind::GitLabCi => validate_gitlab_ci_template(yaml), - } -} + let mode = if dry_run { + gunbc_exec::ExecutionMode::DryRun(gunbc_exec::BoundaryMocks::default()) + } else { + gunbc_exec::ExecutionMode::Real + }; -fn validate_required_sections(yaml: &str, required: &[&str]) -> Result<(), String> { - for section in required { - if !yaml.contains(section) { - return Err(format!("missing required section: {section}")); + match gunbc_exec::execute_with_mode_and_inputs(&dag, mode, None) { + Ok(log) => { + // Report results from the execution log + let github_written = log + .get("tools.cigen::cigen") + .and_then(|entry| entry.outputs.get("github_written")) + .and_then(|v| v.as_bool()); + let gitlab_written = log + .get("tools.cigen::cigen") + .and_then(|entry| entry.outputs.get("gitlab_written")) + .and_then(|v| v.as_bool()); + + let github_status = match (dry_run, github_written) { + (true, _) => "dry-run", + (_, Some(true)) => "written", + (_, Some(false)) => "unchanged", + _ => "ok", + }; + let gitlab_status = match (dry_run, gitlab_written) { + (true, _) => "dry-run", + (_, Some(true)) => "written", + (_, Some(false)) => "unchanged", + _ => "ok", + }; + + println!(" [ci] .github/workflows/ci.yml ({})", github_status); + println!(" [ci] .gitlab-ci.yml ({})", gitlab_status); + println!(); + println!("Generated: 2 CI files"); } - } - Ok(()) -} - -fn validate_github_actions_template(yaml: &str) -> Result<(), String> { - validate_required_sections( - yaml, - &[ - "name:", - "on:", - "permissions:", - "env:", - "jobs:", - "runs-on:", - "steps:", - ], - )?; - - // Basic interpolation sanity check to catch malformed template insertion. - let opens = yaml.matches("${{").count(); - let closes = yaml.matches("}}").count(); - if opens != closes { - return Err(format!( - "unbalanced GitHub interpolation markers: {} opening vs {} closing", - opens, closes - )); - } - - Ok(()) -} - -fn validate_gitlab_ci_template(yaml: &str) -> Result<(), String> { - validate_required_sections( - yaml, - &["image:", "variables:", "stages:", "cache:", "script:"], - )?; - - Ok(()) -} - -/// Generate GitHub Actions YAML template. -fn generate_github_actions_template(config: &RenderConfig) -> String { - let mut yaml = String::new(); - - yaml.push_str(&config.header("#")); - write!(yaml, "\n\nname: {}\n\n", config.workflow_name).unwrap(); - - let branches = config.git.ci_branches(); - yaml.push_str("on:\n push:\n"); - yaml_block(&mut yaml, " branches:", &branches, |b| { - format!(" - {}", b) - }); - yaml.push_str(" pull_request:\n"); - yaml_block(&mut yaml, " branches:", &branches, |b| { - format!(" - {}", b) - }); - - yaml_block( - &mut yaml, - "permissions:", - &config.permissions, - |(scope, level)| format!(" {}: {}", scope, level), - ); - - yaml_block(&mut yaml, "env:", &config.all_env(), |(k, v)| { - format!(" {}: {}", k, v) - }); - - write!( - yaml, - "jobs:\n {}:\n runs-on: {}\n timeout-minutes: {}\n steps:\n", - config.workflow_name, config.runner.id, config.timeout_minutes, - ) - .unwrap(); - - if let Some(checkout) = &config.checkout { - yaml.push_str(" - name: Checkout\n uses: actions/checkout@v4\n"); - if let Some(depth) = checkout.fetch_depth { - write!(yaml, " with:\n fetch-depth: {}\n", depth).unwrap(); + Err(e) => { + print_attention( + AttentionLevel::Error, + "CI generation failed", + &e.to_string(), + ); + std::process::exit(1); } - yaml.push('\n'); - } - - yaml.push_str(" - name: Setup Rust\n uses: dtolnay/rust-toolchain@stable\n\n"); - - if let Some(cache) = &config.cache { - yaml.push_str(" - name: Cache Cargo\n uses: actions/cache@v4\n with:\n"); - yaml_block(&mut yaml, " path: |", &cache.paths, |p| { - format!(" {}", p) - }); - writeln!(yaml, " key: {}", cache.key).unwrap(); - yaml_block( - &mut yaml, - " restore-keys: |", - &cache.restore_keys, - |k| format!(" {}", k), - ); - } - - yaml.push_str( - " - name: Verify Bootstrap Invariants\n run: |\n rm -rf target/codegen\n # Cargo validates all [[bin]] paths even with --bin filter.\n # Create minimal stubs so the manifest parses, then check only bootstrap binaries.\n for dir in $(grep 'path = \"../target/codegen/' gunbc-dag/Cargo.toml | sed 's|.*\"../\\(.*\\)/main.rs\"|\\1|'); do\n mkdir -p \"$dir\" && echo 'fn main() {}' > \"$dir/main.rs\"\n done\n cargo check -p gunbc-dag --bin gunbc-codegen --bin gunbc-ci\n\n", - ); - - write!( - yaml, - " - name: Run CI Pipeline\n run: {}\n", - config.tool.command(), - ) - .unwrap(); - - // Step-level env: CARGO_INCREMENTAL overrides dtolnay/rust-toolchain's - // CARGO_INCREMENTAL=0 (set via $GITHUB_ENV). Step-level env takes precedence. - yaml.push_str(" env:\n"); - yaml.push_str(" CARGO_INCREMENTAL: \"1\"\n"); - for secret in &config.secrets_env { - writeln!(yaml, " {}: ${{{{ secrets.{} }}}}", secret, secret).unwrap(); } - - yaml -} - -/// Generate GitLab CI YAML template. -fn generate_gitlab_ci_template(config: &RenderConfig) -> String { - let mut yaml = String::new(); - - yaml.push_str(&config.header("#")); - yaml.push_str("\n\nimage: rust:latest\n\n"); - - yaml_block(&mut yaml, "variables:", &config.all_env(), |(k, v)| { - format!(" {}: \"{}\"", k, v) - }); - - yaml.push_str("stages:\n - ci\n\n"); - - yaml.push_str( - "cache:\n key: cargo-${CI_COMMIT_REF_SLUG}\n paths:\n - .cargo/\n - target/\n\n", - ); - - write!( - yaml, - "{}:\n stage: ci\n script:\n - {}\n", - config.workflow_name, - config.tool.command(), - ) - .unwrap(); - - yaml } /// Resolve workspace package names to their directory paths using cargo metadata. @@ -1106,11 +913,7 @@ fn print_help() { #[cfg(test)] mod tests { - use super::{ - discover_codegen_tools, generate_github_actions_template, generate_gitlab_ci_template, - parse_command_arg, validate_generated_ci_template, CiTemplateKind, WorkspaceBinary, - }; - use gunbc_ir::transport::ci::{CacheConfig, RenderConfig}; + use super::{discover_codegen_tools, parse_command_arg}; use std::collections::BTreeSet; use std::path::PathBuf; @@ -1137,53 +940,19 @@ mod tests { assert!(err.contains("unexpected extra positional arguments")); } + /// CI YAML generation is now DSL-driven (CG-4). + /// Validation of the rendered templates happens via the DSL compiler + /// (type checking) and the cigen DSL graph test in dsl_builder::tests. #[test] - fn github_template_passes_static_validation() { - let codegen = WorkspaceBinary::Codegen.invocation(); - let config = RenderConfig::new("ci", WorkspaceBinary::Ci.invocation()) - .with_generator(&codegen.binary, &format!("{} -- cigen", codegen.command())) - .with_runner(gunbc_ir::transport::github_actions::ubuntu_latest()) - .with_cargo_env(gunbc_ir::CargoEnv::ci()) - .with_cache(CacheConfig::rust()) - .with_permissions(vec![ - ("contents".to_string(), "read".to_string()), - ("id-token".to_string(), "write".to_string()), - ]); - - let yaml = generate_github_actions_template(&config); - validate_generated_ci_template(CiTemplateKind::GitHubActions, &yaml) - .expect("generated GitHub Actions template should validate"); - } - - #[test] - fn github_template_validation_rejects_missing_sections() { - let malformed = "name: ci\njobs:\n"; - let err = validate_generated_ci_template(CiTemplateKind::GitHubActions, malformed) - .expect_err("malformed GitHub template should fail validation"); - assert!(err.contains("missing required section")); + fn cigen_dsl_graph_builds_successfully() { + let dag = gunbc_dag::dsl_builder::build_dsl_graph_for_entrypoint( + "tools/cigen.dag", + Some("cigen"), + ) + .expect("cigen DSL graph should compile and resolve"); + assert!(!dag.nodes.is_empty()); } - #[test] - fn gitlab_template_passes_static_validation() { - let codegen = WorkspaceBinary::Codegen.invocation(); - let config = RenderConfig::new("ci", WorkspaceBinary::Ci.invocation()) - .with_generator(&codegen.binary, &format!("{} -- cigen", codegen.command())) - .with_runner(gunbc_ir::transport::github_actions::ubuntu_latest()) - .with_cargo_env(gunbc_ir::CargoEnv::ci()) - .with_cache(CacheConfig::rust()); - - let yaml = generate_gitlab_ci_template(&config); - validate_generated_ci_template(CiTemplateKind::GitLabCi, &yaml) - .expect("generated GitLab CI template should validate"); - } - - #[test] - fn gitlab_template_validation_rejects_missing_sections() { - let malformed = "image: rust:latest\n"; - let err = validate_generated_ci_template(CiTemplateKind::GitLabCi, malformed) - .expect_err("malformed GitLab template should fail validation"); - assert!(err.contains("missing required section")); - } #[test] fn codegen_discovery_finds_expected_tools() { let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/gunbc-dag/src/dsl_builder.rs b/gunbc-dag/src/dsl_builder.rs index c1244c2cf3f..e7138b136da 100644 --- a/gunbc-dag/src/dsl_builder.rs +++ b/gunbc-dag/src/dsl_builder.rs @@ -381,4 +381,11 @@ mod tests { ); } + #[test] + fn builds_cigen_dsl_graph() { + let dag = build_dsl_graph_for_entrypoint("tools/cigen.dag", Some("cigen")) + .expect("cigen DSL graph should resolve (CG-3)"); + assert!(!dag.nodes.is_empty()); + } + } diff --git a/gunbc-dag/src/extern_impls.rs b/gunbc-dag/src/extern_impls.rs index e2f7819f990..05846bd4702 100644 --- a/gunbc-dag/src/extern_impls.rs +++ b/gunbc-dag/src/extern_impls.rs @@ -28,6 +28,7 @@ pub fn all_extern_symbols() -> &'static [(&'static str, &'static str)] { ("std.markdown", "render_tree"), ("tools.bootstrap", "render_bootstrap_gitignore"), ("tools.bootstrap", "render_bootstrap_makefile"), + ("tools.cigen", "discover_ci_config"), ("tools.gist", "build_snapshot_content"), ("tools.makegen", "discover_tools"), ("tools.pragma", "render_clippy_toml"), @@ -42,6 +43,8 @@ pub fn lookup_extern_impl(module: &str, name: &str) -> Option { match (module, name) { ("std.markdown", "render_tree") => Some(DynOp::new(RenderTreeOp)), + ("tools.cigen", "discover_ci_config") => Some(DynOp::new(DiscoverCiConfigOp)), + ("tools.gist", "build_snapshot_content") => Some(DynOp::new(BuildSnapshotContentOp)), ("tools.makegen", "discover_tools") => Some(DynOp::new(DiscoverToolsOp)), @@ -176,6 +179,140 @@ fn extract_file_contents(inputs: &HashMap) -> Result, } } +// ============================================================================ +// tools.cigen extern impls +// ============================================================================ + +/// `discover_ci_config() -> CiConfig` +/// +/// Builds a CiConfig record from the repo's CI workflow configuration: +/// permissions, secrets, env, runner, cache, and tool command. +#[derive(Debug, Clone)] +struct DiscoverCiConfigOp; + +impl Executable for DiscoverCiConfigOp { + fn execute( + &self, + _inputs: HashMap, + ) -> Result, ExecError> { + use crate::ci::{ci_live_test_secrets, ci_workflow_permissions}; + use crate::WorkspaceBinary; + use gunbc_ir::transport::github_actions::ubuntu_latest; + + let codegen = WorkspaceBinary::Codegen.invocation(); + let tool = WorkspaceBinary::Ci.invocation(); + let runner = ubuntu_latest(); + + // Permissions from CI workflow integrations + let permissions: Vec = ci_workflow_permissions() + .into_iter() + .map(|(scope, level)| { + let mut map = BTreeMap::new(); + map.insert( + "scope".to_string(), + Value::Str(scope.as_yaml_key().to_string()), + ); + // Map permission levels to DSL sum type variant names + let level_variant = match level.as_yaml_value() { + "read" => "PermRead", + "write" => "PermWrite", + _ => "PermNone", + }; + map.insert("level".to_string(), Value::Str(level_variant.to_string())); + Value::Map(map) + }) + .collect(); + + // Secrets from testgen metadata + let secrets: Vec = ci_live_test_secrets() + .into_iter() + .map(|s| Value::Str(s.to_string())) + .collect(); + + // Standard Rust CI env (derived from CargoEnv::ci()) + let cargo_env = gunbc_ir::CargoEnv::ci(); + let env: Vec = cargo_env + .to_env_map() + .into_iter() + .map(|(k, v)| { + let mut map = BTreeMap::new(); + map.insert("key".to_string(), Value::Str(k)); + map.insert("value".to_string(), Value::Str(v)); + Value::Map(map) + }) + .collect(); + + // Cache configuration + let cache = { + let mut map = BTreeMap::new(); + map.insert( + "key".to_string(), + Value::Str( + "cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}".to_string(), + ), + ); + map.insert( + "paths".to_string(), + Value::List(vec![ + Value::Str("~/.cargo/bin/".to_string()), + Value::Str("~/.cargo/registry/index/".to_string()), + Value::Str("~/.cargo/registry/cache/".to_string()), + Value::Str("~/.cargo/git/db/".to_string()), + ]), + ); + map.insert( + "restore_keys".to_string(), + Value::List(vec![Value::Str("cargo-${{ runner.os }}-".to_string())]), + ); + Value::Map(map) + }; + + // Checkout (default) + let checkout = { + let mut map = BTreeMap::new(); + map.insert("fetch_depth".to_string(), Value::Unit); + map.insert("submodules".to_string(), Value::Unit); + Value::Map(map) + }; + + // Branches from git config + let git = gunbc_ir::GitConfig::default(); + let branches: Vec = git + .ci_branches() + .into_iter() + .map(|b| Value::Str(b.to_string())) + .collect(); + + // Build the CiConfig record + let mut config = BTreeMap::new(); + config.insert("workflow_name".to_string(), Value::Str("ci".to_string())); + config.insert("runner".to_string(), Value::Str(runner.id.to_string())); + config.insert("timeout_minutes".to_string(), Value::Int(30)); + config.insert("branches".to_string(), Value::List(branches)); + config.insert("permissions".to_string(), Value::List(permissions)); + config.insert("env".to_string(), Value::List(env)); + config.insert("cache".to_string(), cache); + config.insert("checkout".to_string(), checkout); + config.insert("secrets".to_string(), Value::List(secrets)); + config.insert( + "tool_command".to_string(), + Value::Str(tool.command().to_string()), + ); + config.insert( + "generator_name".to_string(), + Value::Str(codegen.binary.clone()), + ); + config.insert( + "regenerate_command".to_string(), + Value::Str(format!("{} -- cigen", codegen.command())), + ); + + OutputMap::new() + .value("return", Value::Map(config)) + .ok() + } +} + // ============================================================================ // tools.makegen extern impls // ============================================================================ @@ -611,6 +748,30 @@ mod tests { } } + #[test] + fn test_discover_ci_config() { + let result = DiscoverCiConfigOp.execute(HashMap::new()).unwrap(); + let config = result.get("return").expect("expected return key"); + match config { + Value::Map(map) => { + assert_eq!( + map.get("workflow_name").and_then(Value::as_str), + Some("ci") + ); + assert!(map.contains_key("runner"), "config missing runner"); + assert!(map.contains_key("permissions"), "config missing permissions"); + assert!(map.contains_key("env"), "config missing env"); + assert!(map.contains_key("cache"), "config missing cache"); + assert!(map.contains_key("secrets"), "config missing secrets"); + assert!( + map.contains_key("tool_command"), + "config missing tool_command" + ); + } + _ => panic!("expected Map, got {:?}", std::mem::discriminant(config)), + } + } + #[test] fn test_generate_makefile() { let result = GenerateBootstrapMakefileOp.execute(HashMap::new()).unwrap(); diff --git a/gunbc-dag/tests/extern_ratchet.rs b/gunbc-dag/tests/extern_ratchet.rs index 8ea180f1f44..ccf65cdd5e3 100644 --- a/gunbc-dag/tests/extern_ratchet.rs +++ b/gunbc-dag/tests/extern_ratchet.rs @@ -9,11 +9,12 @@ use gunbc_dag::extern_impls::all_extern_symbols; use std::path::PathBuf; /// Current baseline: number of `extern func` declarations in all `.dag` files. -const EXTERN_FUNC_DECL_BASELINE: usize = 2; +/// Increased from 2 → 3: added `discover_ci_config()` for DSL-driven cigen (CG-3). +const EXTERN_FUNC_DECL_BASELINE: usize = 3; /// Current baseline: number of extern implementations in `all_extern_symbols()`. -/// Decreased from 8 → 6: allowlist + lint_policy migrated to DSL evaluation (FC-P6-d). -const EXTERN_IMPL_BASELINE: usize = 6; +/// Increased from 6 → 7: added `discover_ci_config` for DSL-driven cigen (CG-4). +const EXTERN_IMPL_BASELINE: usize = 7; #[test] #[allow(clippy::disallowed_methods)] diff --git a/tasks.md b/tasks.md index 6320423477a..8352d2bf0f8 100644 --- a/tasks.md +++ b/tasks.md @@ -116,7 +116,7 @@ L4+ needs transport on all services the local profile touches — BT6 handles th | 13 | BT12 | **ArtifactStore providers.** `gcs_artifact_store.dag` (cloud_run) + `inline_artifact_store.dag` (local). Transport blocks, test blocks, profile bindings. | L8 | M | Done | BT10 | | 14 | BT-R2 | **SDLC review: provider completion.** Transport blocks on gcs_claim_store + gcs_outcome_ledger. Inline definitions extracted to provider files. Dead code deleted from pipelines/sdlc.dag. deploy.dag variable naming fixed. Profile bindings updated in sdlc.dag. | — | M | Done | BT11, BT12 | | 15 | BT-R3 | **SDLC review: fix 3 execution gaps.** LLM mock responses (enriched `default_rest_response` with LLM-shaped fields), `navigate_json_path` `/` separator + array index support, auth credential embedding in `GenericRestPrepareOp`, `CallParamSourceOp` replaces `IdentityCallableOp` for param_source nodes, param_source propagation in sdlc.rs CLI. Design: `docs/design/mock-response-pipeline.md`. | — | M | Done | BT10 | -| 16 | BT-E1 | **Transport node deduplication.** `gunbc-sdlc --dry-run` fails at 408/494 nodes: `scalar input 'prepare_transport_...Anthropic_Messages.max_tokens' has multiple upstream edges`. Root cause: lowerer creates ONE shared transport triplet per service operation, but `endpoint_use_count` resets per module — callables in different modules both wire literal sources to the same prepare node's scalar port. Fix: make `endpoint_use_count` global across all modules in the compiled graph (not per-module). Touches: `daglang-lower/src/lib.rs` (`add_service_call_edges`, line 5683). | L1 | M | Pending | BT-R3 | +| 16 | BT-E1 | **Transport node deduplication.** `gunbc-sdlc --dry-run` fails at 408/494 nodes: `scalar input 'prepare_transport_...Anthropic_Messages.max_tokens' has multiple upstream edges`. Root cause: lowerer creates ONE shared transport triplet per service operation, but `endpoint_use_count` resets per module — callables in different modules both wire literal sources to the same prepare node's scalar port. Fix: make `endpoint_use_count` global across all modules in the compiled graph (not per-module). Touches: `daglang-lower/src/lib.rs` (`add_service_call_edges`, line 5683). | L1 | M | Done | BT-R3 | ### Postmortem: Testgen Discovery Bug (BT-R1) @@ -144,7 +144,7 @@ L4+ needs transport on all services the local profile touches — BT6 handles th | CT-1 | **Contract IR.** Parse `@contract` annotations into `ContractObligation` structs in lowerer. Sequence/idempotency/destructive obligation types. Store in type registry alongside interface capabilities. Design: `docs/design/contract-testing.md` §Phase 1. | — | L | BT-R1 | | CT-2 | **Contract test generation.** For each interface with `@contract`, testgen emits parameterized test suite. Suite takes `ServiceBinding` as input. Each obligation becomes a test case: setup → execute sequence → assert postcondition. | — | L | CT-1 | | CT-3 | **Provider compliance wiring.** For each (profile, interface, provider) triple, instantiate CT-2 suite. Stub providers: fast/hermetic/always-run. Real providers: env-gated/integration profiles. Wire into existing PT-* infrastructure. | — | M | CT-2 | -| CT-4 | **Annotation cleanup (Category 3).** Delete metadata noise annotations (`@network`, `@credential`, `@external`, `@derived_from`, `@ledger`, ~30 uses) per `docs/design/modeling/annotation-to-dag-modeling.md` Category 3. | — | S | — | +| CT-4 | **Annotation cleanup (Category 3).** Delete metadata noise annotations (`@network`, `@credential`, `@external`, `@derived_from`, `@ledger`, ~30 uses) per `docs/design/modeling/annotation-to-dag-modeling.md` Category 3. | — | S | Done — annotation infrastructure removed when compiler switched to typed syntax (ba1ce0b). | **Deliverable**: `gunbc sdlc --profile local --repo owner/name` runs full lifecycle. **Endstate**: SDLC on Cloud Run with GCS stores, PubSub signals, multi-worker CAS. @@ -180,10 +180,10 @@ Triaged and sized. Promote to lane queues when horizon items are exhausted. | ID | Item | Size | Priority | Notes | |----|------|------|----------|-------| -| CG-1 | DSL CI model types: `dsl/std/ci.dag` — `CiWorkflow`, `CiJob`, `CiStep` (Run/Uses/DagRun), `CiTrigger`, `CiPermission`, `CiCache`, `CiEnv`, plus provider sum type `CiProvider = GitHub \| GitLab`. Data declarations for shared configs (Rust cache paths, cargo env). | M | P1 | Layer 0 types — no rendering yet. Follow `std/languages.dag` pattern for tautological definitions. | -| CG-2 | DSL CI rendering functions: `dsl/std/ci_render.dag` — `render_github_workflow(w: CiWorkflow) -> String`, `render_gitlab_workflow(w: CiWorkflow) -> String`, plus helpers (`render_step`, `render_job`, `render_permissions`, `render_env_block`, `render_cache`). Pure functions, string interpolation + join. | M | P1 | Follow `makegen.dag` rendering pattern: small composable fns, `\|> map` + `\|> join("\n")`. YAML indentation via string literals (no general YAML serializer needed). | -| CG-3 | DSL cigen tool: `dsl/tools/cigen.dag` — single entrypoint `func cigen() -> { written: Bool }` that discovers CI config via extern (permissions, secrets, tool invocation, branches), constructs `CiWorkflow` records, renders both providers, calls `content_upsert` for each. Extern bridge: `discover_ci_config() -> CiConfig`. | M | P1 | Follow `makegen.dag` entrypoint pattern. Discovery extern returns structured config, all rendering is pure DSL. | -| CG-4 | Delete Rust cigen code: remove `generate_github_actions_template()`, `generate_gitlab_ci_template()`, `validate_github_actions_template()`, `validate_gitlab_ci_template()` from `codegen_cli.rs`. Wire `cmd_cigen()` to the new DSL tool (same pattern as `cmd_codegen()` calling `build_dsl_graph_for_entrypoint`). | S | P1 | ~200 lines deleted from `codegen_cli.rs:450-609`. Validation moves to DSL-side (structural — if the types construct, the YAML is valid). | +| CG-1 | DSL CI model types: `dsl/std/ci.dag` — `CiWorkflow`, `CiJob`, `CiStep` (Run/Uses/DagRun), `CiTrigger`, `CiPermission`, `CiCache`, `CiEnv`, plus provider sum type `CiProvider = GitHub \| GitLab`. Data declarations for shared configs (Rust cache paths, cargo env). | M | P1 | Done | +| CG-2 | DSL CI rendering functions: `dsl/std/ci_render.dag` — `render_github_workflow(config: CiConfig) -> String`, `render_gitlab_ci(config: CiConfig) -> String`, plus helpers. Pure functions, string interpolation + join. | M | P1 | Done | +| CG-3 | DSL cigen tool: `dsl/tools/cigen.dag` — entrypoint `func cigen() -> { github_written: Bool, gitlab_written: Bool }` + Rust extern bridge `discover_ci_config() -> CiConfig`. | M | P1 | Done | +| CG-4 | Delete Rust cigen code: removed `generate_github_actions_template()`, `generate_gitlab_ci_template()`, `validate_*` from `codegen_cli.rs`. Wired `cmd_cigen()` to DSL tool via `build_dsl_graph_for_entrypoint`. ~200 lines deleted. | S | P1 | Done | | CG-5 | Migrate `RenderConfig` builder + `SharedStep` + `yaml_block` from `core/ir/src/transport/ci/render.rs` — evaluate what remains needed as Rust runtime vs what becomes dead code after CG-1:4. Delete dead code, keep only provider detection (`detect_provider`, `is_ci`). | S | P1 | May keep `CiRenderer` trait for runtime step-level rendering (animated progress). CI YAML generation is a separate concern. | | H10 | Compute stack orchestration: Cloud Run/GCS/LB lifecycle DAG builder. | L | P2 | `docs/design/horizon/h10-compute-stack-services.md` | | S12-E | Multi-worker CAS: GcsClaimStore with generation-based CAS. DSL exists. Distinct from B-12 (which stress-tests SignalStore/ArtifactStore). | M | P2 | Deferred until cloud_run profile needed |