From 948d65841610e73ba94aa67e158b5077a2d168e7 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 16:52:08 +0000 Subject: [PATCH 01/11] retire pipeline include_str side channel ratchet --- .../integration/l1_5_fixed_point_test.rs | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index ac256746971..05c89e5383e 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -6,26 +6,35 @@ use std::path::{Path, PathBuf}; use v3_compiler::dag::{ArrowBody, TypeConnective}; use v3_compiler::{ - compare_stage_snapshots, compile_stage_snapshots, default_fixed_point_source, parse_for_test, - tokenize_for_test, Dag, + compare_stage_snapshots, compile_stage_snapshots, default_fixed_point_source, Dag, }; /// Matches `pipeline_authority::PIPELINE_AUTHORITY_FILE` — integration tests cannot import `pub(crate)` helpers. const PIPELINE_AUTHORITY_FILE: &str = "src/v3/compiler/pipeline.dag"; #[test] -fn pipeline_dag_parses() { - let source = include_str!("../../pipeline.dag"); - let tokens = tokenize_for_test(source, "pipeline.dag") - .unwrap_or_else(|diag| panic!("tokenize pipeline.dag failed: {diag:?}")); - let _module = parse_for_test(&tokens, "pipeline.dag") - .unwrap_or_else(|diag| panic!("parse pipeline.dag failed: {diag:?}")); +fn pipeline_dag_bootstrap_authority_is_loaded_structurally() { + let dag = Dag::new(); + let pipeline_types = [ + "PipelineStageBinding", + "PipelineSnapshotKind", + "CompilerHostRealization", + ]; + for name in pipeline_types { + let decl = dag + .declaration_by_name(name) + .unwrap_or_else(|| panic!("pipeline authority declaration `{name}` present")); + assert_eq!( + decl.span.file.as_str(), + PIPELINE_AUTHORITY_FILE, + "`{name}` must come from the bootstrapped pipeline authority" + ); + } } /// Ratchet for `bridge_include_str_side_channels_retired` (pipeline slice): library sources under -/// `src/v3/compiler/src/` must not embed `pipeline.dag` via `include_str!` — ordering authority -/// stays structural (`PipelineStageBinding` / bootstrap witness). Integration tests may still -/// `include_str!("../../pipeline.dag")` for parse fixtures (`pipeline_dag_parses`). +/// `src/v3/compiler/src/` and compiler tests must not embed `pipeline.dag` via `include_str!` — +/// ordering authority stays structural (`PipelineStageBinding` / bootstrap witness). fn collect_rs_files(dir: &Path, out: &mut Vec) { let entries = fs::read_dir(dir).unwrap_or_else(|e| panic!("read_dir {} failed: {e}", dir.display())); @@ -41,13 +50,15 @@ fn collect_rs_files(dir: &Path, out: &mut Vec) { } #[test] -fn compiler_src_has_no_include_str_pipeline_dag_authority() { +fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let src_dir = manifest_dir.join("src"); let mut files = Vec::new(); - collect_rs_files(&src_dir, &mut files); + collect_rs_files(&manifest_dir.join("src"), &mut files); + collect_rs_files(&manifest_dir.join("tests"), &mut files); files.sort(); let mut offenders = Vec::new(); + let forbidden_macro = concat!("include", "_str!"); + let forbidden_path = concat!("pipeline", ".dag"); for path in files { let text = fs::read_to_string(&path) .unwrap_or_else(|e| panic!("read {} failed: {e}", path.display())); @@ -56,7 +67,7 @@ fn compiler_src_has_no_include_str_pipeline_dag_authority() { if trimmed.starts_with("//") { continue; } - if line.contains("include_str!") && line.contains("pipeline.dag") { + if line.contains(forbidden_macro) && line.contains(forbidden_path) { offenders.push(format!( "{}:{}:{}", path.strip_prefix(&manifest_dir).unwrap_or(&path).display(), @@ -68,8 +79,10 @@ fn compiler_src_has_no_include_str_pipeline_dag_authority() { } assert!( offenders.is_empty(), - "`src/**/*.rs` must not use include_str! on pipeline.dag (see bridge_ledger \ + "`src/**/*.rs` and `tests/**/*.rs` must not use {} on {} (see bridge_ledger \ bridge_include_str_side_channels_retired + pipeline_authority.rs). Offenders:\n{}", + forbidden_macro, + forbidden_path, offenders.join("\n") ); } From 37c5e234177521e2779da979e82e3edb2f6e4c2a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 13:32:43 -0400 Subject: [PATCH 02/11] WIP: R3 gate #34: bridge_include_str_side_channels_retired (T-Bridge-Retireme --- .../integration/l1_5_fixed_point_test.rs | 96 ++++++++++++++++--- 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 05c89e5383e..4b590c1b697 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -49,6 +49,67 @@ fn collect_rs_files(dir: &Path, out: &mut Vec) { } } +fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &str) -> Vec { + let forbidden_macro = concat!("include", "_str!"); + let forbidden_path = concat!("pipeline", ".dag"); + let mut offenders = Vec::new(); + for (idx, _) in text.match_indices(forbidden_macro) { + let line_start = text[..idx].rfind('\n').map_or(0, |pos| pos + 1); + let line = &text[line_start..text[idx..].find('\n').map_or(text.len(), |pos| idx + pos)]; + let trimmed = line.trim_start(); + if trimmed.starts_with("//") { + continue; + } + + let after_macro = idx + forbidden_macro.len(); + let Some(open_offset) = text[after_macro..].find('(') else { + continue; + }; + if !text[after_macro..after_macro + open_offset] + .chars() + .all(char::is_whitespace) + { + continue; + } + let open = after_macro + open_offset; + let mut depth = 0usize; + let mut close = None; + for (pos, ch) in text[open..].char_indices() { + match ch { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + close = Some(open + pos); + break; + } + } + _ => {} + } + } + let Some(close) = close else { + continue; + }; + + let macro_arg = &text[open + 1..close]; + let joined_literals: String = macro_arg + .split('"') + .skip(1) + .step_by(2) + .collect::>() + .join(""); + if macro_arg.contains(forbidden_path) || joined_literals.contains(forbidden_path) { + offenders.push(format!( + "{}:{}:{}", + path.strip_prefix(manifest_dir).unwrap_or(path).display(), + text[..idx].bytes().filter(|byte| *byte == b'\n').count() + 1, + trimmed + )); + } + } + offenders +} + #[test] fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -62,20 +123,11 @@ fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { for path in files { let text = fs::read_to_string(&path) .unwrap_or_else(|e| panic!("read {} failed: {e}", path.display())); - for (idx, line) in text.lines().enumerate() { - let trimmed = line.trim_start(); - if trimmed.starts_with("//") { - continue; - } - if line.contains(forbidden_macro) && line.contains(forbidden_path) { - offenders.push(format!( - "{}:{}:{}", - path.strip_prefix(&manifest_dir).unwrap_or(&path).display(), - idx + 1, - trimmed - )); - } - } + offenders.extend(include_str_pipeline_dag_offenders( + &manifest_dir, + &path, + &text, + )); } assert!( offenders.is_empty(), @@ -87,6 +139,22 @@ fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { ); } +#[test] +fn include_str_pipeline_dag_ratchet_catches_split_literals() { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let path = manifest_dir.join("tests/integration/synthetic.rs"); + let synthetic = format!( + r#" +const OK: &str = include{}("other.dag"); +const BAD: &str = include{}(concat!("../../", "pipeline", ".dag")); +"#, + "_str!", "_str!" + ); + let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); + assert_eq!(offenders.len(), 1); + assert!(offenders[0].contains("synthetic.rs:3:")); +} + #[test] fn bootstrap_loads_pipeline_stage_realizations() { let dag = Dag::new(); From f508979155c72df8e42c025a41a8d250963cb11e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 18:32:42 +0000 Subject: [PATCH 03/11] clean up pipeline include_str scanner --- .../integration/l1_5_fixed_point_test.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 4b590c1b697..828f019db19 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -11,6 +11,8 @@ use v3_compiler::{ /// Matches `pipeline_authority::PIPELINE_AUTHORITY_FILE` — integration tests cannot import `pub(crate)` helpers. const PIPELINE_AUTHORITY_FILE: &str = "src/v3/compiler/pipeline.dag"; +const FORBIDDEN_PIPELINE_DAG_MACRO: &str = concat!("include", "_str!"); +const FORBIDDEN_PIPELINE_DAG_PATH: &str = concat!("pipeline", ".dag"); #[test] fn pipeline_dag_bootstrap_authority_is_loaded_structurally() { @@ -50,10 +52,8 @@ fn collect_rs_files(dir: &Path, out: &mut Vec) { } fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &str) -> Vec { - let forbidden_macro = concat!("include", "_str!"); - let forbidden_path = concat!("pipeline", ".dag"); let mut offenders = Vec::new(); - for (idx, _) in text.match_indices(forbidden_macro) { + for (idx, _) in text.match_indices(FORBIDDEN_PIPELINE_DAG_MACRO) { let line_start = text[..idx].rfind('\n').map_or(0, |pos| pos + 1); let line = &text[line_start..text[idx..].find('\n').map_or(text.len(), |pos| idx + pos)]; let trimmed = line.trim_start(); @@ -61,7 +61,7 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s continue; } - let after_macro = idx + forbidden_macro.len(); + let after_macro = idx + FORBIDDEN_PIPELINE_DAG_MACRO.len(); let Some(open_offset) = text[after_macro..].find('(') else { continue; }; @@ -78,6 +78,9 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s match ch { '(' => depth += 1, ')' => { + if depth == 0 { + break; + } depth -= 1; if depth == 0 { close = Some(open + pos); @@ -98,7 +101,9 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s .step_by(2) .collect::>() .join(""); - if macro_arg.contains(forbidden_path) || joined_literals.contains(forbidden_path) { + if macro_arg.contains(FORBIDDEN_PIPELINE_DAG_PATH) + || joined_literals.contains(FORBIDDEN_PIPELINE_DAG_PATH) + { offenders.push(format!( "{}:{}:{}", path.strip_prefix(manifest_dir).unwrap_or(path).display(), @@ -118,8 +123,6 @@ fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { collect_rs_files(&manifest_dir.join("tests"), &mut files); files.sort(); let mut offenders = Vec::new(); - let forbidden_macro = concat!("include", "_str!"); - let forbidden_path = concat!("pipeline", ".dag"); for path in files { let text = fs::read_to_string(&path) .unwrap_or_else(|e| panic!("read {} failed: {e}", path.display())); @@ -133,8 +136,8 @@ fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { offenders.is_empty(), "`src/**/*.rs` and `tests/**/*.rs` must not use {} on {} (see bridge_ledger \ bridge_include_str_side_channels_retired + pipeline_authority.rs). Offenders:\n{}", - forbidden_macro, - forbidden_path, + FORBIDDEN_PIPELINE_DAG_MACRO, + FORBIDDEN_PIPELINE_DAG_PATH, offenders.join("\n") ); } From b336684a77370238fd0d17bd007fbbc3484d4afc Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 18:50:20 +0000 Subject: [PATCH 04/11] document pipeline include_str sentinels --- src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 828f019db19..29be61c0e0b 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -11,6 +11,7 @@ use v3_compiler::{ /// Matches `pipeline_authority::PIPELINE_AUTHORITY_FILE` — integration tests cannot import `pub(crate)` helpers. const PIPELINE_AUTHORITY_FILE: &str = "src/v3/compiler/pipeline.dag"; +// Split these sentinel strings so the ratchet does not match its own source. const FORBIDDEN_PIPELINE_DAG_MACRO: &str = concat!("include", "_str!"); const FORBIDDEN_PIPELINE_DAG_PATH: &str = concat!("pipeline", ".dag"); From 5c21923d67379f699d0c33f4a919381cc89edb3f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 21:42:55 +0000 Subject: [PATCH 05/11] Harden include_str pipeline ratchet scanner --- .../integration/l1_5_fixed_point_test.rs | 186 ++++++++++++++++-- 1 file changed, 170 insertions(+), 16 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 29be61c0e0b..cec38361de0 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -12,7 +12,7 @@ use v3_compiler::{ /// Matches `pipeline_authority::PIPELINE_AUTHORITY_FILE` — integration tests cannot import `pub(crate)` helpers. const PIPELINE_AUTHORITY_FILE: &str = "src/v3/compiler/pipeline.dag"; // Split these sentinel strings so the ratchet does not match its own source. -const FORBIDDEN_PIPELINE_DAG_MACRO: &str = concat!("include", "_str!"); +const FORBIDDEN_PIPELINE_DAG_MACRO_NAME: &str = concat!("include", "_str"); const FORBIDDEN_PIPELINE_DAG_PATH: &str = concat!("pipeline", ".dag"); #[test] @@ -52,30 +52,161 @@ fn collect_rs_files(dir: &Path, out: &mut Vec) { } } +fn code_position_mask(text: &str) -> Vec { + let bytes = text.as_bytes(); + let mut mask = vec![true; bytes.len()]; + let mut idx = 0; + + while idx < bytes.len() { + match bytes[idx] { + b'/' if bytes.get(idx + 1) == Some(&b'/') => { + let start = idx; + idx += 2; + while idx < bytes.len() && bytes[idx] != b'\n' { + idx += 1; + } + mask[start..idx].fill(false); + } + b'/' if bytes.get(idx + 1) == Some(&b'*') => { + let start = idx; + idx += 2; + let mut depth = 1usize; + while idx < bytes.len() && depth > 0 { + if bytes[idx] == b'/' && bytes.get(idx + 1) == Some(&b'*') { + depth += 1; + idx += 2; + } else if bytes[idx] == b'*' && bytes.get(idx + 1) == Some(&b'/') { + depth -= 1; + idx += 2; + } else { + idx += 1; + } + } + mask[start..idx].fill(false); + } + b'b' | b'c' if bytes.get(idx + 1) == Some(&b'"') => { + idx = mask_quoted_literal(bytes, &mut mask, idx, idx + 1); + } + b'b' if bytes.get(idx + 1) == Some(&b'r') => { + if let Some(end) = raw_string_end(bytes, idx + 2) { + mask[idx..end].fill(false); + idx = end; + } else { + idx += 1; + } + } + b'r' => { + if let Some(end) = raw_string_end(bytes, idx + 1) { + mask[idx..end].fill(false); + idx = end; + } else { + idx += 1; + } + } + b'"' => { + idx = mask_quoted_literal(bytes, &mut mask, idx, idx); + } + b'\'' => { + idx = mask_quoted_literal(bytes, &mut mask, idx, idx); + } + _ => idx += 1, + } + } + + mask +} + +fn raw_string_end(bytes: &[u8], mut idx: usize) -> Option { + let hash_start = idx; + while bytes.get(idx) == Some(&b'#') { + idx += 1; + } + if bytes.get(idx) != Some(&b'"') { + return None; + } + let hash_count = idx - hash_start; + idx += 1; + while idx < bytes.len() { + if bytes[idx] == b'"' { + let hashes_match = + (0..hash_count).all(|offset| bytes.get(idx + 1 + offset) == Some(&b'#')); + if hashes_match { + return Some(idx + 1 + hash_count); + } + } + idx += 1; + } + Some(bytes.len()) +} + +fn mask_quoted_literal(bytes: &[u8], mask: &mut [bool], start: usize, quote: usize) -> usize { + let delimiter = bytes[quote]; + let mut idx = quote + 1; + while idx < bytes.len() { + if bytes[idx] == b'\\' { + idx = (idx + 2).min(bytes.len()); + } else if bytes[idx] == delimiter { + idx += 1; + break; + } else { + idx += 1; + } + } + mask[start..idx].fill(false); + idx +} + +fn is_rust_ident_byte(byte: u8) -> bool { + byte == b'_' || byte.is_ascii_alphanumeric() +} + +fn skip_rust_trivia(bytes: &[u8], code_mask: &[bool], mut idx: usize) -> usize { + while idx < bytes.len() + && (bytes[idx].is_ascii_whitespace() || !code_mask.get(idx).copied().unwrap_or(false)) + { + idx += 1; + } + idx +} + fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &str) -> Vec { let mut offenders = Vec::new(); - for (idx, _) in text.match_indices(FORBIDDEN_PIPELINE_DAG_MACRO) { - let line_start = text[..idx].rfind('\n').map_or(0, |pos| pos + 1); - let line = &text[line_start..text[idx..].find('\n').map_or(text.len(), |pos| idx + pos)]; - let trimmed = line.trim_start(); - if trimmed.starts_with("//") { + let code_mask = code_position_mask(text); + let bytes = text.as_bytes(); + for (idx, _) in text.match_indices(FORBIDDEN_PIPELINE_DAG_MACRO_NAME) { + if !code_mask.get(idx).copied().unwrap_or(false) { continue; } - - let after_macro = idx + FORBIDDEN_PIPELINE_DAG_MACRO.len(); - let Some(open_offset) = text[after_macro..].find('(') else { + if idx > 0 && is_rust_ident_byte(bytes[idx - 1]) { continue; - }; - if !text[after_macro..after_macro + open_offset] - .chars() - .all(char::is_whitespace) + } + let after_macro_name = idx + FORBIDDEN_PIPELINE_DAG_MACRO_NAME.len(); + if bytes + .get(after_macro_name) + .is_some_and(|byte| is_rust_ident_byte(*byte)) { continue; } - let open = after_macro + open_offset; + let bang = skip_rust_trivia(bytes, &code_mask, after_macro_name); + if bytes.get(bang) != Some(&b'!') { + continue; + } + let line_start = text[..idx].rfind('\n').map_or(0, |pos| pos + 1); + let line = &text[line_start..text[idx..].find('\n').map_or(text.len(), |pos| idx + pos)]; + let trimmed = line.trim_start(); + + let after_bang = bang + 1; + let open = skip_rust_trivia(bytes, &code_mask, after_bang); + if bytes.get(open) != Some(&b'(') { + continue; + }; let mut depth = 0usize; let mut close = None; for (pos, ch) in text[open..].char_indices() { + let absolute_pos = open + pos; + if !code_mask.get(absolute_pos).copied().unwrap_or(false) { + continue; + } match ch { '(' => depth += 1, ')' => { @@ -137,7 +268,7 @@ fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { offenders.is_empty(), "`src/**/*.rs` and `tests/**/*.rs` must not use {} on {} (see bridge_ledger \ bridge_include_str_side_channels_retired + pipeline_authority.rs). Offenders:\n{}", - FORBIDDEN_PIPELINE_DAG_MACRO, + format_args!("{FORBIDDEN_PIPELINE_DAG_MACRO_NAME}!"), FORBIDDEN_PIPELINE_DAG_PATH, offenders.join("\n") ); @@ -151,12 +282,35 @@ fn include_str_pipeline_dag_ratchet_catches_split_literals() { r#" const OK: &str = include{}("other.dag"); const BAD: &str = include{}(concat!("../../", "pipeline", ".dag")); +const ALSO_BAD: &str = include_str /* trivia */ ! (concat!("../../", "pipeline", ".dag")); "#, "_str!", "_str!" ); let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); - assert_eq!(offenders.len(), 1); + assert_eq!(offenders.len(), 2); assert!(offenders[0].contains("synthetic.rs:3:")); + assert!(offenders[1].contains("synthetic.rs:4:")); +} + +#[test] +fn include_str_pipeline_dag_ratchet_ignores_inert_text() { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let path = manifest_dir.join("tests/integration/inert.rs"); + let synthetic = format!( + r##" +const STRING: &str = "include{}(\"pipeline.dag\")"; +const RAW: &str = r#"include{}("pipeline.dag")"#; +// include{}("pipeline.dag") +/* +include{}("pipeline.dag") +*/ +let _actual = include{}(concat!("../../", "pipeline", ".dag")); +"##, + "_str!", "_str!", "_str!", "_str!", "_str!" + ); + let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); + assert_eq!(offenders.len(), 1); + assert!(offenders[0].contains("inert.rs:8:")); } #[test] From 6fc27faa26da94aad4b84181e9377dc76dff5c68 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 22:46:28 +0000 Subject: [PATCH 06/11] Handle lifetimes in pipeline include ratchet --- .../integration/l1_5_fixed_point_test.rs | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index cec38361de0..15dd03427d1 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -107,7 +107,11 @@ fn code_position_mask(text: &str) -> Vec { idx = mask_quoted_literal(bytes, &mut mask, idx, idx); } b'\'' => { - idx = mask_quoted_literal(bytes, &mut mask, idx, idx); + if apostrophe_starts_lifetime_or_label(bytes, idx) { + idx += 1; + } else { + idx = mask_quoted_literal(bytes, &mut mask, idx, idx); + } } _ => idx += 1, } @@ -116,6 +120,20 @@ fn code_position_mask(text: &str) -> Vec { mask } +fn apostrophe_starts_lifetime_or_label(bytes: &[u8], quote: usize) -> bool { + let Some(next) = bytes.get(quote + 1).copied() else { + return false; + }; + if !(next == b'_' || next.is_ascii_alphabetic()) { + return false; + } + let mut idx = quote + 2; + while bytes.get(idx).is_some_and(|byte| is_rust_ident_byte(*byte)) { + idx += 1; + } + bytes.get(idx) != Some(&b'\'') +} + fn raw_string_end(bytes: &[u8], mut idx: usize) -> Option { let hash_start = idx; while bytes.get(idx) == Some(&b'#') { @@ -313,6 +331,24 @@ let _actual = include{}(concat!("../../", "pipeline", ".dag")); assert!(offenders[0].contains("inert.rs:8:")); } +#[test] +fn include_str_pipeline_dag_ratchet_handles_lifetimes_before_offenders() { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let path = manifest_dir.join("tests/integration/lifetime.rs"); + let synthetic = format!( + r#" +fn lifetime_marker<'a>(input: &'a str) -> &'a str {{ + input +}} +const BAD: &str = include{}(concat!("../../", "pipeline", ".dag")); +"#, + "_str!" + ); + let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); + assert_eq!(offenders.len(), 1); + assert!(offenders[0].contains("lifetime.rs:5:")); +} + #[test] fn bootstrap_loads_pipeline_stage_realizations() { let dag = Dag::new(); From 416bc8476280bcc801a2a21c71527ba723d95ce2 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 18:51:00 -0400 Subject: [PATCH 07/11] WIP: R3 gate #34: bridge_include_str_side_channels_retired (T-Bridge-Retireme --- .../integration/l1_5_fixed_point_test.rs | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 15dd03427d1..99466e822f4 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -215,7 +215,8 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s let after_bang = bang + 1; let open = skip_rust_trivia(bytes, &code_mask, after_bang); - if bytes.get(open) != Some(&b'(') { + let Some((open_delimiter, close_delimiter)) = macro_delimiters(bytes.get(open).copied()) + else { continue; }; let mut depth = 0usize; @@ -226,8 +227,8 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s continue; } match ch { - '(' => depth += 1, - ')' => { + _ if ch == open_delimiter => depth += 1, + _ if ch == close_delimiter => { if depth == 0 { break; } @@ -265,6 +266,15 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s offenders } +fn macro_delimiters(open: Option) -> Option<(char, char)> { + match open? { + b'(' => Some(('(', ')')), + b'[' => Some(('[', ']')), + b'{' => Some(('{', '}')), + _ => None, + } +} + #[test] fn compiler_sources_and_tests_have_no_include_str_pipeline_dag_authority() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -301,13 +311,17 @@ fn include_str_pipeline_dag_ratchet_catches_split_literals() { const OK: &str = include{}("other.dag"); const BAD: &str = include{}(concat!("../../", "pipeline", ".dag")); const ALSO_BAD: &str = include_str /* trivia */ ! (concat!("../../", "pipeline", ".dag")); +const BRACKET_BAD: &str = include_str![concat!("../../", "pipeline", ".dag")]; +const BRACE_BAD: &str = include_str!{concat!("../../", "pipeline", ".dag")}; "#, "_str!", "_str!" ); let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); - assert_eq!(offenders.len(), 2); + assert_eq!(offenders.len(), 4); assert!(offenders[0].contains("synthetic.rs:3:")); assert!(offenders[1].contains("synthetic.rs:4:")); + assert!(offenders[2].contains("synthetic.rs:5:")); + assert!(offenders[3].contains("synthetic.rs:6:")); } #[test] From 0ac4ccc20d0d97e462ace58c30d59ab04ac36c3a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 18:53:57 -0400 Subject: [PATCH 08/11] WIP: R3 gate #34: bridge_include_str_side_channels_retired (T-Bridge-Retireme --- src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 99466e822f4..f6ca4b50e3f 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -312,7 +312,7 @@ const OK: &str = include{}("other.dag"); const BAD: &str = include{}(concat!("../../", "pipeline", ".dag")); const ALSO_BAD: &str = include_str /* trivia */ ! (concat!("../../", "pipeline", ".dag")); const BRACKET_BAD: &str = include_str![concat!("../../", "pipeline", ".dag")]; -const BRACE_BAD: &str = include_str!{concat!("../../", "pipeline", ".dag")}; +const BRACE_BAD: &str = include_str!{{concat!("../../", "pipeline", ".dag")}}; "#, "_str!", "_str!" ); From 8b62fb359615d978d9392da00429c021178b34ff Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 23:07:46 +0000 Subject: [PATCH 09/11] Handle raw string fragments in pipeline include ratchet --- .../integration/l1_5_fixed_point_test.rs | 85 +++++++++++++++++-- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index f6ca4b50e3f..d46f992f68e 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -246,12 +246,7 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s }; let macro_arg = &text[open + 1..close]; - let joined_literals: String = macro_arg - .split('"') - .skip(1) - .step_by(2) - .collect::>() - .join(""); + let joined_literals = joined_rust_string_literals(macro_arg); if macro_arg.contains(FORBIDDEN_PIPELINE_DAG_PATH) || joined_literals.contains(FORBIDDEN_PIPELINE_DAG_PATH) { @@ -266,6 +261,76 @@ fn include_str_pipeline_dag_offenders(manifest_dir: &Path, path: &Path, text: &s offenders } +fn joined_rust_string_literals(text: &str) -> String { + let bytes = text.as_bytes(); + let mut joined = String::new(); + let mut idx = 0usize; + while idx < bytes.len() { + if let Some((content_start, content_end, end)) = raw_string_literal_bounds(bytes, idx) { + joined.push_str(&text[content_start..content_end]); + idx = end; + continue; + } + if bytes[idx] == b'"' { + idx += 1; + while idx < bytes.len() { + match bytes[idx] { + b'\\' => { + if let Some(escaped) = bytes.get(idx + 1) { + joined.push(*escaped as char); + idx += 2; + } else { + idx += 1; + } + } + b'"' => { + idx += 1; + break; + } + byte => { + joined.push(byte as char); + idx += 1; + } + } + } + continue; + } + idx += 1; + } + joined +} + +fn raw_string_literal_bounds(bytes: &[u8], start: usize) -> Option<(usize, usize, usize)> { + if bytes.get(start) != Some(&b'r') { + return None; + } + if start > 0 && is_rust_ident_byte(bytes[start - 1]) { + return None; + } + let mut quote = start + 1; + while bytes.get(quote) == Some(&b'#') { + quote += 1; + } + if bytes.get(quote) != Some(&b'"') { + return None; + } + let hash_count = quote - start - 1; + let content_start = quote + 1; + let mut idx = content_start; + while idx < bytes.len() { + if bytes[idx] == b'"' + && idx + 1 + hash_count <= bytes.len() + && bytes[idx + 1..idx + 1 + hash_count] + .iter() + .all(|byte| *byte == b'#') + { + return Some((content_start, idx, idx + 1 + hash_count)); + } + idx += 1; + } + None +} + fn macro_delimiters(open: Option) -> Option<(char, char)> { match open? { b'(' => Some(('(', ')')), @@ -307,21 +372,23 @@ fn include_str_pipeline_dag_ratchet_catches_split_literals() { let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); let path = manifest_dir.join("tests/integration/synthetic.rs"); let synthetic = format!( - r#" + r##" const OK: &str = include{}("other.dag"); const BAD: &str = include{}(concat!("../../", "pipeline", ".dag")); const ALSO_BAD: &str = include_str /* trivia */ ! (concat!("../../", "pipeline", ".dag")); const BRACKET_BAD: &str = include_str![concat!("../../", "pipeline", ".dag")]; const BRACE_BAD: &str = include_str!{{concat!("../../", "pipeline", ".dag")}}; -"#, +const RAW_BAD: &str = include_str!(concat!(r#"../../pipeline"#, r#".dag"#)); +"##, "_str!", "_str!" ); let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); - assert_eq!(offenders.len(), 4); + assert_eq!(offenders.len(), 5); assert!(offenders[0].contains("synthetic.rs:3:")); assert!(offenders[1].contains("synthetic.rs:4:")); assert!(offenders[2].contains("synthetic.rs:5:")); assert!(offenders[3].contains("synthetic.rs:6:")); + assert!(offenders[4].contains("synthetic.rs:7:")); } #[test] From ab4cbbd5d3f78e03e89344cc0cb5e462e61b2429 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 23:26:30 +0000 Subject: [PATCH 10/11] Handle char fragments in pipeline include ratchet --- .../integration/l1_5_fixed_point_test.rs | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index d46f992f68e..77cbe9b37af 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -295,11 +295,35 @@ fn joined_rust_string_literals(text: &str) -> String { } continue; } + if let Some((ch, end)) = char_literal_value(bytes, idx) { + joined.push(ch); + idx = end; + continue; + } idx += 1; } joined } +fn char_literal_value(bytes: &[u8], start: usize) -> Option<(char, usize)> { + if bytes.get(start) != Some(&b'\'') || apostrophe_starts_lifetime_or_label(bytes, start) { + return None; + } + let content = start + 1; + match bytes.get(content).copied()? { + b'\\' => { + let escaped = bytes.get(content + 1).copied()?; + if bytes.get(content + 2) == Some(&b'\'') { + Some((escaped as char, content + 3)) + } else { + None + } + } + byte if bytes.get(content + 1) == Some(&b'\'') => Some((byte as char, content + 2)), + _ => None, + } +} + fn raw_string_literal_bounds(bytes: &[u8], start: usize) -> Option<(usize, usize, usize)> { if bytes.get(start) != Some(&b'r') { return None; @@ -379,16 +403,18 @@ const ALSO_BAD: &str = include_str /* trivia */ ! (concat!("../../", "pipeline", const BRACKET_BAD: &str = include_str![concat!("../../", "pipeline", ".dag")]; const BRACE_BAD: &str = include_str!{{concat!("../../", "pipeline", ".dag")}}; const RAW_BAD: &str = include_str!(concat!(r#"../../pipeline"#, r#".dag"#)); +const CHAR_BAD: &str = include_str!(concat!("../../pipe", 'l', "ine.dag")); "##, "_str!", "_str!" ); let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); - assert_eq!(offenders.len(), 5); + assert_eq!(offenders.len(), 6); assert!(offenders[0].contains("synthetic.rs:3:")); assert!(offenders[1].contains("synthetic.rs:4:")); assert!(offenders[2].contains("synthetic.rs:5:")); assert!(offenders[3].contains("synthetic.rs:6:")); assert!(offenders[4].contains("synthetic.rs:7:")); + assert!(offenders[5].contains("synthetic.rs:8:")); } #[test] From 464537fdb762e9815f1c912511f0f9086b0d1cb4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 14 May 2026 23:51:49 +0000 Subject: [PATCH 11/11] Decode escapes in pipeline include ratchet --- .../integration/l1_5_fixed_point_test.rs | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs index 77cbe9b37af..d3a32cad3df 100644 --- a/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs +++ b/src/v3/compiler/tests/integration/l1_5_fixed_point_test.rs @@ -276,9 +276,9 @@ fn joined_rust_string_literals(text: &str) -> String { while idx < bytes.len() { match bytes[idx] { b'\\' => { - if let Some(escaped) = bytes.get(idx + 1) { - joined.push(*escaped as char); - idx += 2; + if let Some((escaped, end)) = rust_escape_value(bytes, idx) { + joined.push(escaped); + idx = end; } else { idx += 1; } @@ -312,9 +312,9 @@ fn char_literal_value(bytes: &[u8], start: usize) -> Option<(char, usize)> { let content = start + 1; match bytes.get(content).copied()? { b'\\' => { - let escaped = bytes.get(content + 1).copied()?; - if bytes.get(content + 2) == Some(&b'\'') { - Some((escaped as char, content + 3)) + let (escaped, end) = rust_escape_value(bytes, content)?; + if bytes.get(end) == Some(&b'\'') { + Some((escaped, end + 1)) } else { None } @@ -324,6 +324,39 @@ fn char_literal_value(bytes: &[u8], start: usize) -> Option<(char, usize)> { } } +fn rust_escape_value(bytes: &[u8], backslash: usize) -> Option<(char, usize)> { + if bytes.get(backslash) != Some(&b'\\') { + return None; + } + match bytes.get(backslash + 1).copied()? { + b'n' => Some(('\n', backslash + 2)), + b'r' => Some(('\r', backslash + 2)), + b't' => Some(('\t', backslash + 2)), + b'\\' => Some(('\\', backslash + 2)), + b'0' => Some(('\0', backslash + 2)), + b'\'' => Some(('\'', backslash + 2)), + b'"' => Some(('"', backslash + 2)), + b'x' => { + let hex = std::str::from_utf8(bytes.get(backslash + 2..backslash + 4)?).ok()?; + let value = u8::from_str_radix(hex, 16).ok()?; + Some((value as char, backslash + 4)) + } + b'u' if bytes.get(backslash + 2) == Some(&b'{') => { + let mut end = backslash + 3; + while end < bytes.len() && bytes[end] != b'}' { + end += 1; + } + if bytes.get(end) != Some(&b'}') { + return None; + } + let hex = std::str::from_utf8(&bytes[backslash + 3..end]).ok()?; + let value = u32::from_str_radix(hex, 16).ok()?; + Some((char::from_u32(value)?, end + 1)) + } + escaped => Some((escaped as char, backslash + 2)), + } +} + fn raw_string_literal_bounds(bytes: &[u8], start: usize) -> Option<(usize, usize, usize)> { if bytes.get(start) != Some(&b'r') { return None; @@ -404,17 +437,19 @@ const BRACKET_BAD: &str = include_str![concat!("../../", "pipeline", ".dag")]; const BRACE_BAD: &str = include_str!{{concat!("../../", "pipeline", ".dag")}}; const RAW_BAD: &str = include_str!(concat!(r#"../../pipeline"#, r#".dag"#)); const CHAR_BAD: &str = include_str!(concat!("../../pipe", 'l', "ine.dag")); +const ESCAPE_BAD: &str = include_str!(concat!("../../pipe\u{{6c}}ine", ".dag")); "##, "_str!", "_str!" ); let offenders = include_str_pipeline_dag_offenders(manifest_dir, &path, &synthetic); - assert_eq!(offenders.len(), 6); + assert_eq!(offenders.len(), 7); assert!(offenders[0].contains("synthetic.rs:3:")); assert!(offenders[1].contains("synthetic.rs:4:")); assert!(offenders[2].contains("synthetic.rs:5:")); assert!(offenders[3].contains("synthetic.rs:6:")); assert!(offenders[4].contains("synthetic.rs:7:")); assert!(offenders[5].contains("synthetic.rs:8:")); + assert!(offenders[6].contains("synthetic.rs:9:")); } #[test]