From 67611481ae4ca37817806b076b0bdff0200fa92f Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Thu, 30 Jul 2026 00:24:27 +0000 Subject: [PATCH] refactor: Port Cargo watch and prune knowledge --- crates/turborepo-engine/src/builder/test.rs | 28 -- crates/turborepo-lib/src/commands/prune.rs | 16 +- .../src/package_changes_watcher.rs | 91 +++-- crates/turborepo-repository/src/cargo.rs | 314 ++++++++++-------- .../src/change_knowledge.rs | 95 +++++- crates/turborepo-repository/src/lib.rs | 1 + .../src/package_graph/builder.rs | 64 ++-- .../src/package_graph/mod.rs | 33 +- .../src/prune_knowledge.rs | 65 ++++ crates/turborepo-repository/src/toolchain.rs | 120 ++----- crates/turborepo-scope/src/filter.rs | 14 - crates/turborepo/ARCHITECTURE.md | 4 +- 12 files changed, 459 insertions(+), 386 deletions(-) create mode 100644 crates/turborepo-repository/src/prune_knowledge.rs diff --git a/crates/turborepo-engine/src/builder/test.rs b/crates/turborepo-engine/src/builder/test.rs index cf878d5c5a402..b936ea68c67a9 100644 --- a/crates/turborepo-engine/src/builder/test.rs +++ b/crates/turborepo-engine/src/builder/test.rs @@ -140,20 +140,6 @@ impl Toolchain for AggregateToolchain { )) }) } - - fn watch_spec(&self) -> turborepo_repository::toolchain::WatchSpec { - turborepo_repository::toolchain::WatchSpec::default() - } - - fn prune_plan( - &self, - _kept_packages: &[String], - ) -> Result< - Option, - turborepo_repository::toolchain::Error, - > { - Ok(None) - } } type StubIOEngineResult = Engine; @@ -218,20 +204,6 @@ impl Toolchain for StubIOToolchain { )) }) } - - fn watch_spec(&self) -> turborepo_repository::toolchain::WatchSpec { - turborepo_repository::toolchain::WatchSpec::default() - } - - fn prune_plan( - &self, - _kept_packages: &[String], - ) -> Result< - Option, - turborepo_repository::toolchain::Error, - > { - Ok(None) - } } fn stub_io_package_graph( diff --git a/crates/turborepo-lib/src/commands/prune.rs b/crates/turborepo-lib/src/commands/prune.rs index 8908df042d55f..501df56cf1823 100644 --- a/crates/turborepo-lib/src/commands/prune.rs +++ b/crates/turborepo-lib/src/commands/prune.rs @@ -85,6 +85,8 @@ pub enum Error { PackageNotPruneable(String), #[error(transparent)] Toolchain(#[from] turborepo_repository::toolchain::Error), + #[error(transparent)] + PruneKnowledge(#[from] turborepo_repository::prune_knowledge::Error), } static ADDITIONAL_FILES: LazyLock)>> = @@ -289,16 +291,14 @@ pub async fn prune( } } - // Each toolchain contributes whatever the pruned repository needs - // beyond the packages themselves: extra members it requires, rewritten - // workspace files, and config files to carry over. - for toolchain in prune.package_graph.toolchains().iter() { - let toolchain_id = toolchain.id(); - let kept = kept_by_toolchain.remove(&toolchain_id).unwrap_or_default(); - let Some(plan) = toolchain.prune_plan(&kept)? else { + // Project plans from immutable knowledge captured by this graph's + // discovery generation; live toolchains retain no prune authority. + for toolchain_id in prune.package_graph.prune_toolchains() { + let kept = kept_by_toolchain.remove(toolchain_id).unwrap_or_default(); + let Some(plan) = prune.package_graph.prune_plan(toolchain_id, &kept)? else { continue; }; - planned_toolchains.insert(toolchain_id); + planned_toolchains.insert(toolchain_id.clone()); for extra in plan.extra_packages { let name = PackageName::Other(extra.clone()); let context = prune.package_context(&name)?; diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index efa702f82b711..137e294213307 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -3,7 +3,10 @@ use std::{ collections::{HashMap, HashSet}, io::ErrorKind, ops::DerefMut, - sync::{Arc, RwLock}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, RwLock, + }, }; use notify::Event; @@ -125,6 +128,7 @@ struct Subscriber { repo_root: AbsoluteSystemPathBuf, repository_ignore: RepositoryIgnore, watch_spec: Arc>, + watch_spec_ready: Arc, package_change_events_tx: broadcast::Sender, hash_watcher: Arc, custom_turbo_json_path: Option, @@ -251,11 +255,10 @@ fn classify_changed_files( // Whether an anchored path is under one of the toolchains' // build-byproduct directories. let in_ignored_prefix = |path: &AnchoredSystemPathBuf| { - watch_spec.ignore_prefixes.iter().any(|prefix| { - path.components() - .next() - .is_some_and(|component| component.as_str() == prefix) - }) + watch_spec + .ignore_prefixes + .iter() + .any(|prefix| path_is_under_prefix(path, prefix)) }; // Toolchain workspace-definition files (e.g. Cargo manifests and the @@ -330,6 +333,15 @@ fn classify_changed_files( } } +fn path_is_under_prefix(path: &AnchoredSystemPath, prefix: &str) -> bool { + let path = path.to_unix(); + path.as_str() == prefix + || path + .as_str() + .strip_prefix(prefix) + .is_some_and(|suffix| suffix.starts_with('/')) +} + impl RepoState { fn get_change_mapper(&self) -> Option>> { let Ok(package_change_mapper) = GlobalDepsPackageChangeMapper::new( @@ -398,19 +410,16 @@ impl Subscriber { let repository_ignore = file_events .repository_ignore() .unwrap_or_else(|| RepositoryIgnore::new(repo_root.as_std_path())); - let mut watch_spec = WatchSpec::default(); - if !single_package { - for toolchain in &extra_toolchains { - watch_spec.extend(toolchain.watch_spec()); - } - } - Subscriber { repo_root, file_events, changed_files: Default::default(), repository_ignore, - watch_spec: Arc::new(RwLock::new(watch_spec)), + watch_spec: Arc::new(RwLock::new(WatchSpec::default())), + // Before the first graph generation is published, retain every + // in-repository event. This closes the discovery/subscription race + // without consulting live toolchains for bootstrap facts. + watch_spec_ready: Arc::new(AtomicBool::new(false)), package_change_events_tx, hash_watcher, custom_turbo_json_path: normalized_custom_path, @@ -506,6 +515,7 @@ impl Subscriber { .watch_spec .write() .unwrap_or_else(|poisoned| poisoned.into_inner()) = pkg_dep_graph.active_watch_spec(); + self.watch_spec_ready.store(true, Ordering::Release); Some(RepoState { root_turbo_json, @@ -595,6 +605,7 @@ impl Subscriber { let repo_root = self.repo_root.clone(); let repository_ignore = self.repository_ignore.clone(); let watch_spec = self.watch_spec.clone(); + let watch_spec_ready = self.watch_spec_ready.clone(); let gitignore_path = repo_root.join_component(".gitignore"); let config_paths = [ repo_root.join_component(CONFIG_FILE), @@ -620,14 +631,16 @@ impl Subscriber { let Ok(path) = repo_root.anchor(&absolute_path) else { return false; }; + if !watch_spec_ready.load(Ordering::Acquire) { + return true; + } let watch_spec = watch_spec .read() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let in_ignored_prefix = watch_spec.ignore_prefixes.iter().any(|prefix| { - path.components() - .next() - .is_some_and(|component| component.as_str() == prefix) - }); + let in_ignored_prefix = watch_spec + .ignore_prefixes + .iter() + .any(|prefix| path_is_under_prefix(&path, prefix)); if in_ignored_prefix { return false; } @@ -1001,6 +1014,14 @@ mod test { .await } + fn cargo_watch_spec() -> WatchSpec { + WatchSpec { + definition_file_names: vec!["Cargo.toml".to_string()], + definition_paths: vec!["Cargo.lock".to_string()], + ignore_prefixes: vec!["target".to_string()], + } + } + #[tokio::test(flavor = "multi_thread")] async fn initializes_pure_cargo_and_preserves_root_and_aggregate_scopes() { let tmp = tempfile::tempdir().unwrap(); @@ -1016,10 +1037,7 @@ mod test { .expect("native toolchain permits an absent root package.json"); assert!(!state.pkg_dep_graph.has_root_javascript_scope()); assert!(state.root_turbo_json.is_some()); - assert_eq!( - state.pkg_dep_graph.active_watch_spec(), - turborepo_repository::cargo::watch_spec() - ); + assert_eq!(state.pkg_dep_graph.active_watch_spec(), cargo_watch_spec()); let scopes: Vec<_> = hash_scopes(&state.pkg_dep_graph).collect(); assert_eq!(scopes.first().unwrap().name, PackageName::Root); @@ -1418,8 +1436,7 @@ mod test { // Manifests define the crate set and its edges; the watcher's graph // is stale after any manifest change. - let action = - f.classify_with_spec(&trie, &[], None, turborepo_repository::cargo::watch_spec()); + let action = f.classify_with_spec(&trie, &[], None, cargo_watch_spec()); assert!(matches!(action, FileChangeAction::ConfigChanged)); // Without the Cargo toolchain registered, the same file is ordinary @@ -1435,8 +1452,7 @@ mod test { let mut trie = Trie::new(); trie.insert(lock.to_string(), ()); - let action = - f.classify_with_spec(&trie, &[], None, turborepo_repository::cargo::watch_spec()); + let action = f.classify_with_spec(&trie, &[], None, cargo_watch_spec()); assert!(matches!(action, FileChangeAction::ConfigChanged)); } @@ -1459,8 +1475,7 @@ mod test { (), ); - let action = - f.classify_with_spec(&trie, &[], None, turborepo_repository::cargo::watch_spec()); + let action = f.classify_with_spec(&trie, &[], None, cargo_watch_spec()); assert!( matches!(action, FileChangeAction::NoRelevantChanges), "target/ writes must be dropped, got {action:?}" @@ -1472,6 +1487,24 @@ mod test { assert!(matches!(action, FileChangeAction::PackagesChanged(..))); } + #[tokio::test] + async fn classify_nested_target_directory_prefix_writes_ignored() { + let f = ClassifyFixture::new().await; + let mut trie = Trie::new(); + trie.insert( + f.repo_root + .join_components(&["build", "cargo", "debug", "app"]) + .to_string(), + (), + ); + let spec = WatchSpec { + ignore_prefixes: vec!["build/cargo".to_string()], + ..WatchSpec::default() + }; + let action = f.classify_with_spec(&trie, &[], None, spec); + assert!(matches!(action, FileChangeAction::NoRelevantChanges)); + } + #[tokio::test] async fn classify_custom_turbo_json_triggers_config_changed() { let f = ClassifyFixture::new().await; diff --git a/crates/turborepo-repository/src/cargo.rs b/crates/turborepo-repository/src/cargo.rs index 0de66d87cba1e..75787204c6fe0 100644 --- a/crates/turborepo-repository/src/cargo.rs +++ b/crates/turborepo-repository/src/cargo.rs @@ -42,11 +42,13 @@ use serde::Deserialize; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; use crate::{ + change_knowledge::ChangeObservation, external_resolution::{ ExternalPackageIdentity, ExternalResolutionData, ExternalResolutionDomain, PackageResolution, ResolutionCompleteness, ResolutionFingerprint, }, package_json::{DependencyKind, PackageJson}, + prune_knowledge::{PruneDomain, PrunePlan}, relationships::Relationship, toolchain::{ self, DiscoverPackagesFuture, DiscoveredPackage, DiscoveredPackages, Toolchain, @@ -351,9 +353,7 @@ pub enum CargoPackageKind { Workspace, } -/// Cargo-specific details for a discovered package, retained by the -/// [`CargoToolchain`] (keyed by package name) rather than attached to the -/// toolchain-neutral `PackageInfo`. +/// Cargo-specific details captured in immutable task-contract knowledge. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CargoPackageDetails { pub kind: CargoPackageKind, @@ -361,9 +361,6 @@ pub struct CargoPackageDetails { /// workspace aggregate). pub deliverables: Vec, pub manifest_alters_output_layout: bool, - /// The crate's directory, repo-root-relative in unix form (empty for - /// the workspace aggregate). - pub dir: String, } const VERIFICATION_SUBCOMMANDS: &[(&str, &str)] = &[ @@ -1179,37 +1176,123 @@ fn cargo_output_layout( }) } +fn cargo_change_observation( + repo_root: &AbsoluteSystemPath, + target_directory: Option<&AbsoluteSystemPath>, +) -> ChangeObservation { + let mut observation = ChangeObservation::new(ToolchainId::RUST) + .with_rediscovery_file_name(CARGO_TOML) + .with_resolution_path(CARGO_LOCK); + if let Some(prefix) = target_directory + .and_then(|path| repo_root.anchor(path).ok()) + .filter(|path| path.components().next().is_some()) + { + observation = observation.with_ignore_prefix(prefix.to_unix().to_string()); + } + observation +} + +/// Cargo prune inputs captured atomically with the discovery generation. +#[derive(Debug)] +struct CargoPruneKnowledge { + toolchain: ToolchainId, + lockfile: String, + root_manifest: String, + package_directories: HashMap, +} + +impl CargoPruneKnowledge { + fn discover(repo_root: &AbsoluteSystemPath, crates: &[CargoCrate]) -> Result { + let lockfile = match repo_root.join_component(CARGO_LOCK).read_to_string() { + Ok(contents) => contents, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err(Error::MissingLockfile); + } + Err(error) => return Err(Error::LockfileRead(error)), + }; + let root_manifest = repo_root + .join_component(CARGO_TOML) + .read_to_string() + .map_err(Error::WorkspaceFileRead)?; + let package_directories = crates + .iter() + .filter_map(|cargo_crate| { + let directory = cargo_crate.manifest_path.parent()?; + let directory = AnchoredSystemPathBuf::new(repo_root, directory).ok()?; + Some((cargo_crate.name.clone(), directory.to_unix().to_string())) + }) + .collect(); + Ok(Self { + toolchain: ToolchainId::RUST, + lockfile, + root_manifest, + package_directories, + }) + } +} + +impl PruneDomain for CargoPruneKnowledge { + fn toolchain(&self) -> &ToolchainId { + &self.toolchain + } + + fn plan( + &self, + kept_packages: &[String], + ) -> Result, crate::prune_knowledge::Error> { + if kept_packages.is_empty() { + return Ok(None); + } + let failed = |error: Error| crate::prune_knowledge::Error::Failed(Box::new(error)); + let pruned_lock = turborepo_lockfiles::cargo_prune_lock(&self.lockfile, kept_packages) + .map_err(|error| failed(Error::Lockfile(error)))?; + + let mut kept_dirs = Vec::with_capacity(pruned_lock.members.len()); + let mut extra_packages = Vec::new(); + for member in &pruned_lock.members { + let Some(directory) = self.package_directories.get(member) else { + tracing::warn!( + "Cargo.lock member {member} is not a discovered workspace crate; skipping" + ); + continue; + }; + kept_dirs.push(directory.clone()); + if !kept_packages.contains(member) { + extra_packages.push(member.clone()); + } + } + let pruned_manifest = + prune_root_manifest(&self.root_manifest, &kept_dirs).map_err(failed)?; + Ok(Some(PrunePlan { + extra_packages, + root_files: vec![ + (CARGO_LOCK.to_string(), pruned_lock.lockfile), + (CARGO_TOML.to_string(), pruned_manifest), + ], + copy_paths: [ + "rust-toolchain.toml", + "rust-toolchain", + ".cargo/config.toml", + ".cargo/config", + ] + .into_iter() + .map(str::to_string) + .collect(), + })) + } +} + /// The Cargo toolchain. Registered in the /// [`crate::toolchain::ToolchainRegistry`] when /// `futureFlags.experimentalCargoWorkspaces` is enabled and the repository /// root contains a `Cargo.toml`. pub struct CargoToolchain { repo_root: AbsoluteSystemPathBuf, - /// Per-package details retained for entrypoint selection and pruning. - details: std::sync::Mutex>, } impl CargoToolchain { pub fn new(repo_root: AbsoluteSystemPathBuf) -> Arc { - Arc::new(Self { - repo_root, - details: std::sync::Mutex::new(HashMap::new()), - }) - } - - fn package_details(&self, package: &str) -> Option { - self.details - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .get(package) - .cloned() - } - - fn record_details(&self, package: String, details: CargoPackageDetails) { - self.details - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .insert(package, details); + Arc::new(Self { repo_root }) } } @@ -1291,87 +1374,6 @@ impl Toolchain for CargoToolchain { vars } - fn watch_spec(&self) -> toolchain::WatchSpec { - watch_spec() - } - - /// Prune the Cargo workspace machinery around the kept crates: - /// - /// * `Cargo.lock` is subset to the closure of the kept crates, so `cargo - /// build --locked` succeeds in the pruned output. - /// * The lock walk may surface members beyond Turborepo's package-graph - /// closure (Cargo.lock merges dev-dependency edges, including - /// cycle-participating ones the package graph drops). Their manifests are - /// referenced by kept crates, so they are reported as extra packages to - /// keep. - /// * The root `Cargo.toml` is rewritten: explicit `members`, filtered - /// `default-members`, `[workspace.dependencies]` path entries to removed - /// crates dropped. - /// * Toolchain and Cargo config files are carried over. - fn prune_plan( - &self, - kept_packages: &[String], - ) -> Result, toolchain::Error> { - if kept_packages.is_empty() { - return Ok(None); - } - let failed = |err: Error| toolchain::Error::Failed(Box::new(err)); - - let lock_path = self.repo_root.join_component(CARGO_LOCK); - let lock_contents = match lock_path.read_to_string() { - Ok(contents) => contents, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - return Err(failed(Error::MissingLockfile)); - } - Err(error) => return Err(failed(Error::LockfileRead(error))), - }; - let pruned_lock = turborepo_lockfiles::cargo_prune_lock(&lock_contents, kept_packages) - .map_err(|err| failed(Error::Lockfile(err)))?; - - let mut kept_dirs = Vec::with_capacity(pruned_lock.members.len()); - let mut extra_packages = Vec::new(); - for member in &pruned_lock.members { - let Some(details) = self.package_details(member) else { - // A lock member that discovery never saw; the lockfile and - // the workspace disagree. Keep going — the manifest rewrite - // simply won't list it, and cargo will report specifics. - tracing::warn!( - "Cargo.lock member {member} is not a discovered workspace crate; skipping" - ); - continue; - }; - kept_dirs.push(details.dir.clone()); - if !kept_packages.contains(member) { - extra_packages.push(member.clone()); - } - } - - let manifest_contents = self - .repo_root - .join_component(CARGO_TOML) - .read_to_string() - .map_err(|err| failed(Error::WorkspaceFileRead(err)))?; - let pruned_manifest = - prune_root_manifest(&manifest_contents, &kept_dirs).map_err(failed)?; - - Ok(Some(toolchain::PrunePlan { - extra_packages, - root_files: vec![ - (CARGO_LOCK.to_string(), pruned_lock.lockfile), - (CARGO_TOML.to_string(), pruned_manifest), - ], - copy_paths: [ - "rust-toolchain.toml", - "rust-toolchain", - ".cargo/config.toml", - ".cargo/config", - ] - .iter() - .map(|path| path.to_string()) - .collect(), - })) - } - /// Our lock subset is reachability-based, but Cargo's real resolution /// is feature-aware: shrinking the workspace can deactivate features /// that were the only reason some packages were in the closure. Rather @@ -1450,6 +1452,11 @@ impl Toolchain for CargoToolchain { .name .ok_or_else(|| toolchain::Error::Failed(Box::new(Error::MissingWorkspaceName)))?; + let change_observation = + cargo_change_observation(&self.repo_root, target_directory.as_deref()); + let prune_domain = CargoPruneKnowledge::discover(&self.repo_root, &crates) + .map_err(|error| toolchain::Error::Failed(Box::new(error)))?; + // Each crate contributes its already-classified native internal // relationships directly. No JavaScript dependency descriptor or // package-manager policy participates in Cargo graph assembly. @@ -1504,19 +1511,10 @@ impl Toolchain for CargoToolchain { } else { CargoPackageKind::Library }; - let dir = cargo_crate - .manifest_path - .parent() - .and_then(|dir| { - turbopath::AnchoredSystemPathBuf::new(&self.repo_root, dir).ok() - }) - .map(|dir| dir.to_unix().to_string()) - .unwrap_or_default(); let details = CargoPackageDetails { kind, deliverables: cargo_crate.deliverables, manifest_alters_output_layout: cargo_crate.manifest_alters_output_layout, - dir, }; let native_tasks = native_tasks_for_package(&details, &cargo_crate.name); let task_contract = CargoTaskContract::new( @@ -1524,7 +1522,6 @@ impl Toolchain for CargoToolchain { details.clone(), workspace_contract_details.clone(), ); - self.record_details(cargo_crate.name.clone(), details); let external_dependencies: HashSet = closures .remove(&cargo_crate.name) .unwrap_or_default() @@ -1559,7 +1556,6 @@ impl Toolchain for CargoToolchain { kind: CargoPackageKind::Workspace, deliverables: Vec::new(), manifest_alters_output_layout: false, - dir: String::new(), }; let workspace_native_tasks = native_tasks_for_package(&workspace_package_details, &workspace_name); @@ -1568,7 +1564,6 @@ impl Toolchain for CargoToolchain { workspace_package_details.clone(), workspace_contract_details.clone(), ); - self.record_details(workspace_name.clone(), workspace_package_details); crate_names.sort(); let relationships = crate_names .into_iter() @@ -1606,7 +1601,9 @@ impl Toolchain for CargoToolchain { }, ); Ok(DiscoveredPackages::new(packages, workspace_roots) - .with_external_resolution(resolution)) + .with_external_resolution(resolution) + .with_change_observation(change_observation) + .with_prune_domain(Arc::new(prune_domain))) }) } } @@ -1614,23 +1611,6 @@ impl Toolchain for CargoToolchain { /// The Cargo default build directory, relative to the repo root. pub const TARGET_DIR: &str = "target"; -/// How filesystem events relate to Cargo in watch mode. Manifests and the -/// lockfile define the crate set and its edges — any change makes the -/// watcher's package graph stale, so they trigger full rediscovery -/// (`Cargo.toml` files under `target/` are build byproducts, not workspace -/// definition, and are exempted via the ignore prefix). Events under the -/// root `target/` directory are dropped entirely: Cargo writes there -/// continuously during builds, and letting those events through would -/// re-trigger the very tasks that produced them — usually `target/` is -/// gitignored, but a feedback loop must not depend on a `.gitignore` entry. -pub fn watch_spec() -> toolchain::WatchSpec { - toolchain::WatchSpec { - definition_file_names: vec![CARGO_TOML.to_string()], - definition_paths: vec![CARGO_LOCK.to_string()], - ignore_prefixes: vec![TARGET_DIR.to_string()], - } -} - /// Whether `name` is a valid Cargo crate name for our purposes. Cargo itself /// enforces this for published crates; local manifests are looser, so guard /// against names that would break downstream task identifiers. @@ -2302,7 +2282,6 @@ mod test { kind, deliverables, manifest_alters_output_layout: false, - dir: "crate".to_string(), }; let deliverable = |name: &str, kind| Deliverable { name: name.to_string(), @@ -2619,7 +2598,6 @@ dependencies = ["lib-a"] kind: DeliverableKind::Bin, }], manifest_alters_output_layout: false, - dir: "crates/app".to_string(), } } @@ -3533,13 +3511,33 @@ release: 1.96.0-nightly\n", let toolchain = CargoToolchain::new(root.clone()); assert_eq!(toolchain.id(), ToolchainId::RUST); - let (packages, roots, resolutions) = + let (packages, roots, resolutions, changes, prune_domains) = toolchain.discover_packages().await.unwrap().into_parts(); assert_eq!(roots.len(), 1); assert_eq!(roots[0].kind(), "cargo"); assert_eq!(roots[0].path(), root.as_ref()); assert_eq!(resolutions.len(), 1); assert_eq!(resolutions[0].toolchain(), &ToolchainId::RUST); + assert_eq!(changes.len(), 1); + assert_eq!(prune_domains.len(), 1); + let prune_plan = prune_domains[0] + .plan(&["app".to_string()]) + .unwrap() + .expect("a retained Cargo crate produces a prune plan"); + assert_eq!( + prune_plan + .root_files + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + [CARGO_LOCK, CARGO_TOML] + ); + assert!( + prune_plan + .copy_paths + .iter() + .any(|path| path == ".cargo/config") + ); assert_eq!(resolutions[0].definition_sources()[0].as_str(), CARGO_LOCK); let ExternalResolutionData::Resolved { completeness, @@ -3609,11 +3607,13 @@ release: 1.96.0-nightly\n", async fn test_cargo_toolchain_empty_without_manifest() { let (_tmp, root) = tempdir_root(); let toolchain = CargoToolchain::new(root); - let (packages, roots, resolutions) = + let (packages, roots, resolutions, changes, prune_domains) = toolchain.discover_packages().await.unwrap().into_parts(); assert!(packages.is_empty()); assert!(roots.is_empty()); assert!(resolutions.is_empty()); + assert!(changes.is_empty()); + assert!(prune_domains.is_empty()); } #[tokio::test(flavor = "multi_thread")] @@ -3622,11 +3622,13 @@ release: 1.96.0-nightly\n", write(&root, &["Cargo.toml"], "[workspace]\nmembers = []\n"); let toolchain = CargoToolchain::new(root); - let (packages, roots, resolutions) = + let (packages, roots, resolutions, changes, prune_domains) = toolchain.discover_packages().await.unwrap().into_parts(); assert!(packages.is_empty()); assert_eq!(roots.len(), 1); assert!(resolutions.is_empty()); + assert!(changes.is_empty()); + assert!(prune_domains.is_empty()); } fn package_info(name: &str) -> crate::package_graph::PackageInfo { @@ -3640,7 +3642,7 @@ release: 1.96.0-nightly\n", #[rustfmt::skip] fn task_context<'a>( - toolchain: &CargoToolchain, + _toolchain: &CargoToolchain, root: &'a AbsoluteSystemPath, name: &str, directory: &'a str, @@ -3651,9 +3653,27 @@ release: 1.96.0-nightly\n", } else { crate::package_graph::PackageTaskContextKind::Package }; - let native_tasks = toolchain - .package_details(name) - .map(|details| native_tasks_for_package(&details, name)); + let cargo_kind = if directory.is_empty() { + CargoPackageKind::Workspace + } else if name == "app" { + CargoPackageKind::Entrypoint + } else { + CargoPackageKind::Library + }; + let deliverables = if cargo_kind == CargoPackageKind::Entrypoint { + vec![Deliverable { + name: name.to_string(), + kind: DeliverableKind::Bin, + }] + } else { + Vec::new() + }; + let details = CargoPackageDetails { + kind: cargo_kind, + deliverables, + manifest_alters_output_layout: false, + }; + let native_tasks = Some(native_tasks_for_package(&details, name)); crate::package_graph::PackageTaskContext::new_for_test_with_native_tasks( name.into(), root, diff --git a/crates/turborepo-repository/src/change_knowledge.rs b/crates/turborepo-repository/src/change_knowledge.rs index c2a5fb3e549d9..e9a360b38e929 100644 --- a/crates/turborepo-repository/src/change_knowledge.rs +++ b/crates/turborepo-repository/src/change_knowledge.rs @@ -5,8 +5,8 @@ //! subscriptions, coalescing, and generation publication. //! //! JavaScript observations are produced from repository knowledge plus the -//! active package manager. Cargo temporarily retains `Toolchain::watch_spec` -//! until its Rust port. +//! active package manager. Native ecosystems contribute observations with +//! their package discovery result. use std::collections::BTreeMap; @@ -22,9 +22,13 @@ pub struct ChangeKnowledge { /// Manifest file names that can change package membership wherever they /// appear (e.g. `package.json` for JavaScript). membership_file_names: Vec, + /// Manifest names whose changes require full package rediscovery. + rediscovery_file_names: Vec, /// Repo-root-relative unix paths that can change package membership /// (e.g. workspace configuration files). membership_paths: Vec, + /// Native definition/resolution paths whose changes require rediscovery. + rediscovery_paths: Vec, /// Repo-root-relative unix paths that can change external resolution /// (e.g. lockfiles). resolution_paths: Vec, @@ -35,11 +39,47 @@ pub struct ChangeKnowledge { package_directories: BTreeMap, } +/// Parser-neutral change facts contributed by one discovery producer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChangeObservation { + toolchain: ToolchainId, + rediscovery_file_names: Vec, + resolution_paths: Vec, + ignore_prefixes: Vec, +} + +impl ChangeObservation { + pub fn new(toolchain: ToolchainId) -> Self { + Self { + toolchain, + rediscovery_file_names: Vec::new(), + resolution_paths: Vec::new(), + ignore_prefixes: Vec::new(), + } + } + + pub fn with_rediscovery_file_name(mut self, name: impl Into) -> Self { + self.rediscovery_file_names.push(name.into()); + self + } + + pub fn with_resolution_path(mut self, path: impl Into) -> Self { + self.resolution_paths.push(path.into()); + self + } + + pub fn with_ignore_prefix(mut self, path: impl Into) -> Self { + self.ignore_prefixes.push(path.into()); + self + } +} + impl ChangeKnowledge { /// Produce JavaScript change observations from repository knowledge. - pub(crate) fn javascript( + pub(crate) fn build( knowledge: &RepositoryKnowledge, package_manager: Option<&PackageManager>, + native: Vec, ) -> Self { let mut membership_file_names = Vec::new(); let mut membership_paths = Vec::new(); @@ -71,13 +111,36 @@ impl ChangeKnowledge { }) .collect(); - Self { + let mut change = Self { membership_file_names, + rediscovery_file_names: Vec::new(), membership_paths, + rediscovery_paths: Vec::new(), resolution_paths, ignore_prefixes: Vec::new(), package_directories, + }; + + for observation in native { + let active = knowledge + .scopes() + .any(|scope| scope.toolchain() == &observation.toolchain); + if !active { + continue; + } + change + .membership_file_names + .extend(observation.rediscovery_file_names.iter().cloned()); + change + .rediscovery_file_names + .extend(observation.rediscovery_file_names); + change + .rediscovery_paths + .extend(observation.resolution_paths.iter().cloned()); + change.resolution_paths.extend(observation.resolution_paths); + change.ignore_prefixes.extend(observation.ignore_prefixes); } + change } pub fn membership_file_names(&self) -> &[String] { @@ -102,7 +165,7 @@ impl ChangeKnowledge { /// Project change knowledge into the watcher `WatchSpec` shape. /// - /// Only workspace-configuration paths and ignore prefixes are projected + /// Only rediscovery paths/names and ignore prefixes are projected /// into rediscovery triggers. Per-package `package.json` and lockfile /// changes continue to flow through `ChangeMapper` / lockfile content /// analysis so we preserve today's affectedness granularity; those facts @@ -110,18 +173,16 @@ impl ChangeKnowledge { /// [`Self::resolution_paths`]. pub fn to_watch_spec(&self) -> WatchSpec { WatchSpec { - definition_file_names: Vec::new(), - definition_paths: self.membership_paths.clone(), + definition_file_names: self.rediscovery_file_names.clone(), + definition_paths: self + .membership_paths + .iter() + .chain(&self.rediscovery_paths) + .cloned() + .collect(), ignore_prefixes: self.ignore_prefixes.clone(), } } - - /// Combine foundational change knowledge with toolchain WatchSpecs. - pub fn combined_watch_spec(&self, toolchain_spec: WatchSpec) -> WatchSpec { - let mut combined = self.to_watch_spec(); - combined.extend(toolchain_spec); - combined - } } #[cfg(test)] @@ -165,7 +226,7 @@ mod tests { #[test] fn empty_repository_has_no_js_triggers() { let knowledge = empty_repository(); - let change = ChangeKnowledge::javascript(&knowledge, None); + let change = ChangeKnowledge::build(&knowledge, None, Vec::new()); assert!(change.membership_file_names().is_empty()); assert!(change.resolution_paths().is_empty()); assert!(change.package_directories().is_empty()); @@ -174,7 +235,7 @@ mod tests { #[test] fn javascript_includes_package_json_and_lockfile() { let knowledge = javascript_repository(); - let change = ChangeKnowledge::javascript(&knowledge, Some(&PackageManager::Npm)); + let change = ChangeKnowledge::build(&knowledge, Some(&PackageManager::Npm), Vec::new()); assert_eq!(change.membership_file_names(), ["package.json"]); assert_eq!(change.resolution_paths(), ["package-lock.json"]); assert!(change.package_directories().contains_key("web")); @@ -186,7 +247,7 @@ mod tests { #[test] fn pnpm_workspace_config_projects_into_watch_spec() { let knowledge = javascript_repository(); - let change = ChangeKnowledge::javascript(&knowledge, Some(&PackageManager::Pnpm)); + let change = ChangeKnowledge::build(&knowledge, Some(&PackageManager::Pnpm), Vec::new()); assert_eq!(change.resolution_paths(), ["pnpm-lock.yaml"]); let watch = change.to_watch_spec(); assert!( diff --git a/crates/turborepo-repository/src/lib.rs b/crates/turborepo-repository/src/lib.rs index ae5f8af9f0215..f4eaffa0d0d9a 100644 --- a/crates/turborepo-repository/src/lib.rs +++ b/crates/turborepo-repository/src/lib.rs @@ -23,6 +23,7 @@ pub mod native_tasks; pub mod package_graph; pub mod package_json; pub mod package_manager; +pub mod prune_knowledge; pub mod relationships; pub mod task_contracts; pub mod toolchain; diff --git a/crates/turborepo-repository/src/package_graph/builder.rs b/crates/turborepo-repository/src/package_graph/builder.rs index 55dafed1a8a42..6393e313df5fa 100644 --- a/crates/turborepo-repository/src/package_graph/builder.rs +++ b/crates/turborepo-repository/src/package_graph/builder.rs @@ -387,6 +387,8 @@ struct BuildState<'a, S, T> { native_relationships: HashMap>, native_external_resolutions: Vec, native_task_observations: Vec, + native_change_observations: Vec, + native_prune_domains: Vec>, /// The root `package.json`, absent for a pure Cargo workspace. See /// [`PackageGraphBuilder::root_package_json`]. root_package_json: Option, @@ -669,6 +671,8 @@ where native_relationships: HashMap::new(), native_external_resolutions: Vec::new(), native_task_observations: Vec::new(), + native_change_observations: Vec::new(), + native_prune_domains: Vec::new(), lockfile, package_manager: None, package_jsons, @@ -788,9 +792,12 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedPackageManage continue; } let output = toolchain.discover_packages().await?; - let (packages, roots, external_resolutions) = output.into_parts(); + let (packages, roots, external_resolutions, changes, prune_domains) = + output.into_parts(); self.native_external_resolutions .extend(external_resolutions); + self.native_change_observations.extend(changes); + self.native_prune_domains.extend(prune_domains); workspace_roots.extend( roots .into_iter() @@ -847,6 +854,8 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedPackageManage native_relationships, native_external_resolutions, native_task_observations, + native_change_observations, + native_prune_domains, root_package_json, lockfile, package_manager, @@ -864,6 +873,8 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedPackageManage native_relationships, native_external_resolutions, native_task_observations, + native_change_observations, + native_prune_domains, root_package_json, lockfile, package_manager, @@ -997,10 +1008,12 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedPackageManage ) .map_err(|error| Error::TaskContracts(error.to_string()))? }); - let change_knowledge = Arc::new(crate::change_knowledge::ChangeKnowledge::javascript( + let change_knowledge = Arc::new(crate::change_knowledge::ChangeKnowledge::build( &knowledge, package_manager.as_ref(), + Vec::new(), )); + let prune_knowledge = Arc::new(crate::prune_knowledge::PruneKnowledge::default()); Ok(PackageGraph { graph: workspace_graph, @@ -1022,6 +1035,7 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedPackageManage native_task_knowledge, task_contract_knowledge, change_knowledge, + prune_knowledge, }) } } @@ -1178,6 +1192,8 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedWorkspaces, T relationship_knowledge, native_external_resolutions, native_task_observations, + native_change_observations, + native_prune_domains, root_package_json, javascript, toolchains, @@ -1194,6 +1210,8 @@ impl<'a, T: PackageDiscovery + Send + Sync> BuildState<'a, ResolvedWorkspaces, T native_relationships: HashMap::new(), native_external_resolutions, native_task_observations, + native_change_observations, + native_prune_domains, root_package_json, lockfile, package_manager, @@ -1365,6 +1383,8 @@ impl BuildState<'_, ResolvedLockfile, T> { knowledge, relationship_knowledge, native_task_observations, + native_change_observations, + native_prune_domains, root_package_json, toolchains, .. @@ -1420,9 +1440,13 @@ impl BuildState<'_, ResolvedLockfile, T> { discovery::Error::Failed(Box::new(Error::TaskContracts(error.to_string()))) })? }); - let change_knowledge = Arc::new(crate::change_knowledge::ChangeKnowledge::javascript( + let change_knowledge = Arc::new(crate::change_knowledge::ChangeKnowledge::build( &knowledge, package_manager.as_ref(), + native_change_observations, + )); + let prune_knowledge = Arc::new(crate::prune_knowledge::PruneKnowledge::new( + native_prune_domains, )); Ok(PackageGraph { @@ -1445,6 +1469,7 @@ impl BuildState<'_, ResolvedLockfile, T> { native_task_knowledge, task_contract_knowledge, change_knowledge, + prune_knowledge, }) } } @@ -1557,17 +1582,6 @@ mod test { fn discover_packages(&self) -> DiscoverPackagesFuture<'_> { Box::pin(async move { Ok(DiscoveredPackages::new(Vec::new(), self.roots.clone())) }) } - - fn watch_spec(&self) -> crate::toolchain::WatchSpec { - crate::toolchain::WatchSpec::default() - } - - fn prune_plan( - &self, - _kept_packages: &[String], - ) -> Result, crate::toolchain::Error> { - Ok(None) - } } struct PackageWithoutRootToolchain { @@ -1591,17 +1605,6 @@ mod test { )) }) } - - fn watch_spec(&self) -> crate::toolchain::WatchSpec { - crate::toolchain::WatchSpec::default() - } - - fn prune_plan( - &self, - _kept_packages: &[String], - ) -> Result, crate::toolchain::Error> { - Ok(None) - } } struct PackageContributingToolchain { @@ -1623,17 +1626,6 @@ mod test { )) }) } - - fn watch_spec(&self) -> crate::toolchain::WatchSpec { - crate::toolchain::WatchSpec::default() - } - - fn prune_plan( - &self, - _kept_packages: &[String], - ) -> Result, crate::toolchain::Error> { - Ok(None) - } } fn custom_package( diff --git a/crates/turborepo-repository/src/package_graph/mod.rs b/crates/turborepo-repository/src/package_graph/mod.rs index 40eb360121838..5be454a5baf78 100644 --- a/crates/turborepo-repository/src/package_graph/mod.rs +++ b/crates/turborepo-repository/src/package_graph/mod.rs @@ -106,6 +106,8 @@ pub struct PackageGraph { task_contract_knowledge: Arc, /// Immutable change knowledge for watch/affectedness classification. change_knowledge: Arc, + /// Immutable native prune domains from the same discovery generation. + prune_knowledge: Arc, } /// The WorkspacePackage. @@ -746,24 +748,21 @@ impl PackageGraph { &self.toolchains } - /// Watch classification facts for this graph: foundational change knowledge - /// (JavaScript membership/workspace triggers and ignore prefixes) combined - /// with active toolchain `WatchSpec`s (Cargo until its Rust port). - /// - /// Registered toolchains that were inactive (notably extras in - /// single-package mode) are omitted from the toolchain contribution. + /// Watch classification facts retained by this graph generation. pub fn active_watch_spec(&self) -> crate::toolchain::WatchSpec { - let active: HashSet<_> = self - .package_task_contexts() - .filter_map(|context| context.toolchain().cloned()) - .collect(); - let mut toolchain_spec = crate::toolchain::WatchSpec::default(); - for toolchain in self.toolchains.iter() { - if active.contains(&toolchain.id()) { - toolchain_spec.extend(toolchain.watch_spec()); - } - } - self.change_knowledge.combined_watch_spec(toolchain_spec) + self.change_knowledge.to_watch_spec() + } + + pub fn prune_toolchains(&self) -> impl Iterator { + self.prune_knowledge.toolchains() + } + + pub fn prune_plan( + &self, + toolchain: &crate::toolchain::ToolchainId, + kept_packages: &[String], + ) -> Result, crate::prune_knowledge::Error> { + self.prune_knowledge.plan(toolchain, kept_packages) } pub fn repo_root(&self) -> &AbsoluteSystemPath { diff --git a/crates/turborepo-repository/src/prune_knowledge.rs b/crates/turborepo-repository/src/prune_knowledge.rs new file mode 100644 index 0000000000000..b7466c9d3c28c --- /dev/null +++ b/crates/turborepo-repository/src/prune_knowledge.rs @@ -0,0 +1,65 @@ +//! Immutable, generation-owned knowledge used to plan native prune output. + +use std::{collections::BTreeMap, fmt::Debug, sync::Arc}; + +use crate::toolchain::ToolchainId; + +/// A toolchain's contribution to a pruned repository. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PrunePlan { + /// Packages that must additionally be retained and copied. + pub extra_packages: Vec, + /// Files to write as `(repo-relative unix path, contents)`. + pub root_files: Vec<(String, String)>, + /// Repo-relative unix paths to copy verbatim when present. + pub copy_paths: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// An ecosystem-specific planning failure. + #[error(transparent)] + Failed(Box), +} + +/// Immutable discovery output capable of projecting a prune plan. +/// +/// Implementations contain only data captured for one repository generation; +/// they are not live toolchains and cannot mutate discovery authority. +pub trait PruneDomain: Debug + Send + Sync { + fn toolchain(&self) -> &ToolchainId; + fn plan(&self, kept_packages: &[String]) -> Result, Error>; +} + +/// All native prune domains retained by a package-graph generation. +#[derive(Debug, Default)] +pub struct PruneKnowledge { + domains: BTreeMap>, +} + +impl PruneKnowledge { + pub(crate) fn new(domains: Vec>) -> Self { + let mut retained = BTreeMap::new(); + for domain in domains { + let replaced = retained.insert(domain.toolchain().clone(), domain); + debug_assert!(replaced.is_none(), "duplicate prune knowledge domain"); + } + Self { domains: retained } + } + + pub fn toolchains(&self) -> impl Iterator { + self.domains.keys() + } + + pub fn plan( + &self, + toolchain: &ToolchainId, + kept_packages: &[String], + ) -> Result, Error> { + self.domains + .get(toolchain) + .map(|domain| domain.plan(kept_packages)) + .transpose() + .map(Option::flatten) + } +} diff --git a/crates/turborepo-repository/src/toolchain.rs b/crates/turborepo-repository/src/toolchain.rs index 5e2a841898817..f99b216b8d198 100644 --- a/crates/turborepo-repository/src/toolchain.rs +++ b/crates/turborepo-repository/src/toolchain.rs @@ -8,8 +8,8 @@ //! (e.g. Cargo) register alongside it in the [`ToolchainRegistry`]. //! //! The trait grows one concern at a time (discovery today; command -//! resolution, derived task inputs/outputs, external-dependency hashing, -//! watch triggers, and prune participation as they are needed), and every +//! resolution, derived task inputs/outputs, and external-dependency hashing +//! as they are needed), and every //! concern must ship with real implementations for every registered //! toolchain. //! @@ -41,8 +41,8 @@ //! Lockfile handling gains a trait surface with external dependency hashing; //! dependency splitting remains JS-native for now. //! - The prune command's JavaScript machinery (lockfile subgraphing, -//! workspace-file rewriting, patches) is its native path rather than a -//! [`Toolchain::prune_plan`] implementation. +//! workspace-file rewriting, patches) remains on its native path rather than +//! the immutable prune-knowledge path. use std::{borrow::Cow, ffi::OsString, fmt, future::Future, pin::Pin, sync::Arc}; @@ -50,10 +50,12 @@ use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; use turborepo_errors::Spanned; use crate::{ + change_knowledge::ChangeObservation, discovery::{self, PackageDiscovery}, external_resolution::ExternalResolutionDomain, package_json::PackageJson, package_manager::PackageManager, + prune_knowledge::PruneDomain, relationships::Relationship, }; @@ -180,14 +182,26 @@ pub struct DiscoveredPackages { packages: Vec, workspace_roots: Vec, external_resolutions: Vec, + change_observations: Vec, + prune_domains: Vec>, } +pub type DiscoveredPackagesParts = ( + Vec, + Vec, + Vec, + Vec, + Vec>, +); + impl DiscoveredPackages { pub fn new(packages: Vec, workspace_roots: Vec) -> Self { Self { packages, workspace_roots, external_resolutions: Vec::new(), + change_observations: Vec::new(), + prune_domains: Vec::new(), } } @@ -196,6 +210,16 @@ impl DiscoveredPackages { self } + pub fn with_change_observation(mut self, observation: ChangeObservation) -> Self { + self.change_observations.push(observation); + self + } + + pub fn with_prune_domain(mut self, domain: Arc) -> Self { + self.prune_domains.push(domain); + self + } + pub fn packages(&self) -> &[DiscoveredPackage] { &self.packages } @@ -204,17 +228,13 @@ impl DiscoveredPackages { &self.workspace_roots } - pub fn into_parts( - self, - ) -> ( - Vec, - Vec, - Vec, - ) { + pub fn into_parts(self) -> DiscoveredPackagesParts { ( self.packages, self.workspace_roots, self.external_resolutions, + self.change_observations, + self.prune_domains, ) } } @@ -386,17 +406,6 @@ pub trait Toolchain: Send + Sync { /// one observation envelope. fn discover_packages(&self) -> DiscoverPackagesFuture<'_>; - /// How filesystem events relate to this toolchain in watch mode: - /// workspace-definition files whose change requires rediscovery, and - /// build-byproduct directories whose events must be ignored. - fn watch_spec(&self) -> WatchSpec; - - /// What `turbo prune` must carry for this toolchain so the pruned - /// repository is self-contained, given the names of this toolchain's - /// packages already selected for the pruned output. `None` means the - /// toolchain contributes nothing beyond the packages themselves. - fn prune_plan(&self, kept_packages: &[String]) -> Result, Error>; - /// Called after the pruned output is fully written, with its root /// directory. Toolchains may polish their own files in place (e.g. /// Cargo canonicalizes the pruned lockfile through `cargo metadata`) and @@ -457,25 +466,7 @@ pub struct CompileCacheEndpoint { /// this marker to the embedded sccache instead of the normal CLI. pub const COMPILE_CACHE_WRAPPER_ENV: &str = "TURBO_SCCACHE_WRAPPER"; -/// A toolchain's contribution to a pruned repository. See -/// [`Toolchain::prune_plan`]. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct PrunePlan { - /// Packages that must additionally be kept and copied, beyond the ones - /// requested (e.g. crates reachable only through dev-dependency edges, - /// whose manifests are referenced by kept crates). - pub extra_packages: Vec, - /// Files to write into the pruned repository: (repo-relative unix path, - /// contents). They define dependency resolution, so they go to the full - /// layer and, in docker mode, the json layer. - pub root_files: Vec<(String, String)>, - /// Repo-relative unix paths of toolchain configuration files to copy - /// verbatim when present (missing ones are skipped). - pub copy_paths: Vec, -} - -/// How filesystem events relate to a toolchain in watch mode. See -/// [`Toolchain::watch_spec`]. +/// Watch classification projected from immutable change knowledge. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct WatchSpec { /// Manifest file names that define the toolchain's workspace membership @@ -500,16 +491,6 @@ pub struct TaskDefaults { pub cache: Option, } -impl WatchSpec { - /// Merge another spec into this one. - pub fn extend(&mut self, other: WatchSpec) { - self.definition_file_names - .extend(other.definition_file_names); - self.definition_paths.extend(other.definition_paths); - self.ignore_prefixes.extend(other.ignore_prefixes); - } -} - /// Platform-aware environment projection for one toolchain's I/O derivation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TaskIOEnvironment { @@ -648,15 +629,6 @@ impl ToolchainRegistry { pub fn iter(&self) -> impl Iterator { self.toolchains.iter().map(AsRef::as_ref) } - - /// The union of every registered toolchain's [`WatchSpec`]. - pub fn watch_spec(&self) -> WatchSpec { - let mut merged = WatchSpec::default(); - for toolchain in self.iter() { - merged.extend(toolchain.watch_spec()); - } - merged - } } impl fmt::Debug for ToolchainRegistry { @@ -762,26 +734,6 @@ impl Toolchain for JavaScriptToolchain

{ ToolchainId::JAVASCRIPT } - fn watch_spec(&self) -> WatchSpec { - // Deliberately nothing: JavaScript workspace redefinition (a new or - // removed package.json, a lockfile change) is caught by the change - // mapper's conservative fallback — unattributable files map to - // "all packages", which triggers rediscovery — and JS build outputs - // land inside package directories where gitignore filtering already - // applies. This is the real answer, not an unimplemented stub. - WatchSpec::default() - } - - fn prune_plan(&self, _kept_packages: &[String]) -> Result, Error> { - // Known debt (see module docs): the prune command's JavaScript - // machinery — lockfile subgraphing, root package.json and - // pnpm-workspace rewriting, patch carrying — is its native code - // path, predating this abstraction. Folding it into this surface - // means restructuring a battle-tested command; until then, the JS - // contribution is deliberately empty here. - Ok(None) - } - fn discover_packages(&self) -> DiscoverPackagesFuture<'_> { Box::pin(async move { use tracing::Instrument; @@ -1132,14 +1084,6 @@ mod tests { fn discover_packages(&self) -> DiscoverPackagesFuture<'_> { Box::pin(async { Ok(DiscoveredPackages::default()) }) } - - fn watch_spec(&self) -> WatchSpec { - WatchSpec::default() - } - - fn prune_plan(&self, _kept_packages: &[String]) -> Result, Error> { - Ok(None) - } } let mut registry = ToolchainRegistry::new(); diff --git a/crates/turborepo-scope/src/filter.rs b/crates/turborepo-scope/src/filter.rs index 844dedc3b11ad..cbfa50a4d27ca 100644 --- a/crates/turborepo-scope/src/filter.rs +++ b/crates/turborepo-scope/src/filter.rs @@ -1002,20 +1002,6 @@ mod test { )) }) } - - fn watch_spec(&self) -> turborepo_repository::toolchain::WatchSpec { - turborepo_repository::toolchain::WatchSpec::default() - } - - fn prune_plan( - &self, - _kept_packages: &[String], - ) -> Result< - Option, - turborepo_repository::toolchain::Error, - > { - Ok(None) - } } /// Make a project resolver with the provided dependencies. Extras is for diff --git a/crates/turborepo/ARCHITECTURE.md b/crates/turborepo/ARCHITECTURE.md index 85360cef249e7..05caf6037514a 100644 --- a/crates/turborepo/ARCHITECTURE.md +++ b/crates/turborepo/ARCHITECTURE.md @@ -222,8 +222,8 @@ The remaining payload deletion phases are explicit: - **Phase 3:** Complete. External resolution lives in the immutable generation; query, prune, hashing, and summaries consume it. `PackageInfo` no longer carries `unresolved_external_dependencies`, `transitive_dependencies`, or `external_deps_hash`, and deferred closure installation is gone. - **Phase 4 (complete):** Native task/command knowledge is an immutable catalog produced at repository construction. JavaScript scripts and Cargo verb tables contribute observations; engine, turbo-json, executor, query, devtools, LSP, and summary consumers read the catalog. `Toolchain::task_command` / `task_display_command` / `authors_task` / `registered_tasks` / `registers_task` / `defines_task` have been deleted — only the JavaScript producer and the LSP unsaved-source adapter parse scripts. - **Phase 5 (in progress):** Task-contract knowledge catalog is produced for JavaScript scopes; engine composition and global `engines` hashing consume it; JS packages are excluded from Toolchain task-I/O environment dispatch. Remaining: fuller hash/framework contract production and final Phase 5 gate. Later: Delete `PackageInfo`, its payload map, and optional-payload compatibility plumbing once all fail-closed consumers use those queries. -- **Phase 6 (complete for JS change knowledge first wave):** Change knowledge is produced at repository construction; watcher classification and scope lockfile probes consume it via `active_watch_spec` / `resolution_paths` rather than ad-hoc package-manager probes. -- **Phase 7 (complete):** JavaScript prune rendering is a distinct pure step (`render_javascript_prune`) producing typed artifacts; `commands/prune.rs` selects closures, performs path-safe layout, and materializes those artifacts without inline lockfile/manifest/patch format interpretation. Golden inventories cover standard and Docker layouts. +- **Phase 6 (complete for JavaScript and Cargo):** Change knowledge is produced at repository construction. Cargo discovery contributes `Cargo.toml` rediscovery names, the `Cargo.lock` resolution/rediscovery path, and the effective in-repository target-directory ignore prefix. `PackageGraph::active_watch_spec` is now a projection of only the immutable facts retained by that graph generation; it never calls live toolchains. Before the first generation is published, the watcher conservatively retains all in-repository events, closing the subscription/bootstrap race without mutable toolchain callbacks. Single-package generations retain no inactive Cargo facts. +- **Phase 7 (complete):** JavaScript prune rendering is a distinct pure step (`render_javascript_prune`) producing typed artifacts; `commands/prune.rs` selects closures, performs path-safe layout, and materializes those artifacts without inline lockfile/manifest/patch format interpretation. Cargo discovery captures an immutable, generation-owned prune domain containing lockfile, root-manifest, and package-directory facts. Cargo lock pruning, extra-member selection, manifest rewriting, and root/config file planning are projected from that graph-owned domain rather than mutable `CargoToolchain` state. Live toolchains remain involved only in post-write finalization pending TURBO-5798. Golden inventories cover standard and Docker layouts. - **Phase 8 (audit complete for owned consumers):** Query/devtools/summary/run/engine/watch/prune task and resolution views consume knowledge catalogs. Remaining `PackageJson` / `PackageInfo` reads are construction entry points, LSP unsaved-buffer adapters, and package-manager detection — tracked for deletion under TURBO-5787 (`PackageInfo` / `JavaScriptToolchain` removal), not deferred silently. Boundary tag diagnostics consume optional authored-name provenance from repository knowledge only when the authored name matches the authoritative identity. Task hashing, run-cache path construction, and run-summary task directories use