diff --git a/Cargo.lock b/Cargo.lock index 09c2e8de2f839..4d03046aa4075 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9064,6 +9064,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", + "sha2", "tempfile", "test-case", "thiserror 2.0.18", diff --git a/apps/docs/content/docs/guides/tools/python.mdx b/apps/docs/content/docs/guides/tools/python.mdx index cbfc93560f535..02a6232be2ff2 100644 --- a/apps/docs/content/docs/guides/tools/python.mdx +++ b/apps/docs/content/docs/guides/tools/python.mdx @@ -94,9 +94,9 @@ When no supported formatter or checker is detected for a role, Turborepo retains | Package | Turbo task | uv command | | ----------------- | ---------- | ------------------------------- | | Member | `format` | `uv format -- ` | -| Member | `check` | `uv check --package=` | +| Member | `check` | `uv check --frozen --package=` | | Workspace package | `format` | `uv format -- ` | -| Workspace package | `check` | `uv check --all-packages` | +| Workspace package | `check` | `uv check --frozen --all-packages` | ### Tool declarations and inheritance @@ -154,7 +154,7 @@ uv run --frozen pytest uv run --frozen --package py-api pytest packages/py-api ``` -Detected quality-tool and fallback `check` commands run serially in the `uv` execution group. Build, fallback format, and member pytest commands can run in parallel. All mapped uv commands default to `cache: false` and run at the repository root. Detected-tool commands use `--frozen`. Turborepo does not invoke uv during discovery and never creates or updates `uv.lock`; refresh it explicitly with `uv lock`. +Detected quality-tool and fallback `check` commands run serially in the `uv` execution group. Build, fallback format, and member pytest commands can run in parallel. Commands run at the repository root. Detected-tool commands use `--frozen`. Turborepo never creates or updates `uv.lock`; refresh it explicitly with `uv lock`. Turborepo passes the member directory to pytest instead of searching for tests itself. Pytest's own collection rules therefore cover conventional `tests/` directories and colocated tests. The workspace command has no target so pytest can collect the repository-wide suite. Pytest's standard exit code `5` is preserved when a selected scope collects no tests. @@ -199,7 +199,9 @@ Additionally, [`turbo query`](/docs/reference/query) can be used to understand y ## Caching behavior -All built-in uv command tasks default to uncached because the uv, Python, tool, and isolated build-backend identities are not yet represented in their hashes. You can opt in with an explicit `cache: true` only when your repository pins the relevant toolchain. +Turborepo identifies the installed uv frontend and the Python interpreter selected by `uv python find`. The identity includes the uv and Python executable content, Python implementation and version, operating system, architecture, libc, variant, and host compatibility details without including installation paths. When both identities are available, detected lint, check, and test tasks are cacheable by default. If identity resolution fails, requires downloading Python, or finds a repository-local uv executable, these tasks remain uncached so graph discovery continues safely. + +Format tasks remain uncached because they mutate source files. Builds using `uv_build` are cacheable when the package's sole build requirement accepts the bundled backend version in the identified uv executable. Other PEP 517 builds remain uncached because uv can resolve isolated backend dependencies independently of `uv.lock`. The fallback `uv check` task remains uncached because uv resolves its bundled checker independently of the workspace lockfile. Turborepo creates task hashes using: @@ -209,8 +211,9 @@ Turborepo creates task hashes using: - Root `pyproject.toml`, `uv.toml`, `.python-version`, `ruff.toml`, `.ruff.toml`, `mypy.ini`, `.mypy.ini`, `pyrightconfig.json`, `pytest.ini`, `.pytest.ini`, `pytest.toml`, `.pytest.toml`, `setup.py`, `setup.cfg`, `tox.ini`, `ty.toml`, and `conftest.py`, when present - Relevant uv and pip environment variables (index selection, resolution mode, Python selection) - The resolved external dependency closure from `uv.lock`, scoped to each member. Root-owned tools conservatively include the workspace closure +- The resolved uv and Python interpreter identities -Automatic inputs exclude `.venv`, `.pytest_cache`, `.ruff_cache`, `.mypy_cache`, `.pyright`, `.ty`, and `__pycache__`. Path-valued uv environment settings and active user or system uv configuration cannot yet be content-hashed safely, so they make automatic inputs untracked and disable caching unless you explicitly configure `cache`. +Automatic inputs exclude `.venv`, `.pytest_cache`, `.ruff_cache`, `.mypy_cache`, `.pyright`, `.ty`, and `__pycache__`. Path-valued uv environment settings, `UV_NO_SYNC`, `UV_NO_PROJECT`, active user or system uv configuration, and any pass-through arguments cannot be content-hashed safely, so they disable automatic caching unless you explicitly configure `cache`. Project-specific hashing inputs must be accounted for manually. This includes: @@ -223,7 +226,7 @@ For a bare `uv build`, Turborepo detects the matching sdist and wheel in the wor ## Watch mode -Changes to any `pyproject.toml` or the root `uv.lock` trigger workspace rediscovery. Watch mode ignores root `.venv/` and `dist/` events and known Python and quality-tool cache directories at the root and member scopes. +Changes to any `pyproject.toml`, the root `uv.lock`, `.python-version`, or `uv.toml` trigger workspace rediscovery. Watch mode ignores root `.venv/` and `dist/` events and known Python and quality-tool cache directories at the root and member scopes. ## Pruning @@ -236,4 +239,4 @@ Changes to any `pyproject.toml` or the root `uv.lock` trigger workspace rediscov - Reachable local path, directory, editable, or virtual dependencies must be discovered workspace members at the same path recorded in `uv.lock`. Other local sources prevent graph construction because Turborepo cannot yet content-hash or prune them safely. - The synthetic workspace package has no directory and cannot be passed to `turbo prune`; prune a member instead. - Automatic tool discovery is limited to Ruff, Black, mypy, ty, Pyright, and pytest. Unsupported tools require normal task configuration; they are not inferred from Python metadata. -- Tool versions, the Python interpreter, and build-backend identities are not yet hashed, so built-in command tasks remain uncached by default. +- Non-bundled isolated build-backend identities are not yet hashed, so those build tasks remain uncached by default. diff --git a/crates/turborepo-repository/Cargo.toml b/crates/turborepo-repository/Cargo.toml index 92f1e95a70050..ae204793c8061 100644 --- a/crates/turborepo-repository/Cargo.toml +++ b/crates/turborepo-repository/Cargo.toml @@ -26,6 +26,7 @@ rust-ini = "0.20.0" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_yaml_ng = { workspace = true } +sha2 = { workspace = true } thiserror = { workspace = true } tokio.workspace = true toml = { workspace = true } diff --git a/crates/turborepo-repository/src/uv.rs b/crates/turborepo-repository/src/uv.rs index 05114ff0eea3d..7f505561c5090 100644 --- a/crates/turborepo-repository/src/uv.rs +++ b/crates/turborepo-repository/src/uv.rs @@ -9,10 +9,9 @@ //! in-process: member globs are expanded against the filesystem and each //! member's `pyproject.toml` is parsed for its identity and dependencies. //! Unlike Cargo (whose membership semantics only `cargo metadata` can -//! answer), uv workspace membership is declarative globs — and requiring -//! the `uv` binary at discovery time would break graph construction on -//! machines that only orchestrate. The `uv` binary is required only to -//! execute tasks. +//! answer), uv workspace membership is declarative globs. Discovery probes uv +//! and its selected Python interpreter when available so command tasks can be +//! cached safely, but neither binary is required to construct the graph. //! //! Buildable packages register `build` (`uv build --package=`), and all //! packages register `format` and `check`. A synthetic package @@ -32,19 +31,22 @@ //! //! External dependencies hash from `uv.lock` per member (see //! [`external_closures`]), scoped to each member's transitive closure, so a -//! dependency bump only invalidates the packages that depend on it. +//! dependency bump only invalidates the packages that depend on it. Resolved +//! uv and Python identities participate in every Python package hash. //! //! Support is experimental and gated behind //! `futureFlags.experimentalPythonWorkspaces`. use std::{ collections::{BTreeMap, HashMap, HashSet}, - io, + io::{self, Read}, + process::Command, str::FromStr as _, sync::Arc, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; use crate::{ @@ -194,12 +196,20 @@ pub fn is_valid_workspace_name(name: &str) -> bool { struct PyProjectManifest { project: Option, #[serde(rename = "build-system")] - build_system: Option, + build_system: Option, #[serde(default, rename = "dependency-groups")] dependency_groups: BTreeMap, tool: Option, } +#[derive(Debug, Default, Deserialize)] +struct BuildSystemTable { + #[serde(default)] + requires: Vec, + #[serde(rename = "build-backend")] + build_backend: Option, +} + #[derive(Debug, Default, Deserialize)] struct ProjectTable { name: Option, @@ -283,6 +293,14 @@ impl PyProjectManifest { self.build_system.is_some() || self.uv().and_then(|uv| uv.package) == Some(true) } + fn bundled_uv_build_requirement(&self) -> Option<&str> { + let build_system = self.build_system.as_ref()?; + (build_system.build_backend.as_deref() == Some("uv_build") + && build_system.requires.len() == 1 + && normalize_name(pep508_name(&build_system.requires[0])?) == "uv-build") + .then(|| build_system.requires[0].as_str()) + } + /// All declared dependency strings, tagged with their semantic role. /// PEP 735 dependency groups can nest `{ include-group = "…" }` tables; /// only the string entries carry package names, and every group is @@ -734,6 +752,7 @@ pub struct UvPackage { pub relationships: Vec, /// Whether uv can build this member as a Python distribution. pub buildable: bool, + bundled_uv_build_requirement: Option, quality_plan: QualityPlan, pytest: Option, } @@ -1052,6 +1071,9 @@ fn connect_packages( manifest_path, relationships: package_relationships, buildable: manifest.is_buildable(), + bundled_uv_build_requirement: manifest + .bundled_uv_build_requirement() + .map(str::to_string), quality_plan: QualityPlan::effective(root_tools, &member_tools), pytest: member_tools.execution(PythonTool::Pytest), } @@ -1082,6 +1104,7 @@ fn uv_command_task( prefix: Vec, suffix: Vec, serial_group: Option, + cacheable: bool, ) -> crate::native_tasks::NativeTask { use crate::native_tasks::{ NativeCommandArguments, NativeCommandProgram, NativeTask, NativeTaskContract, @@ -1107,7 +1130,9 @@ fn uv_command_task( WorkingDirectoryPolicy::RepositoryRoot, ) .with_contract(NativeTaskContract::new( - toolchain::TaskDefaults { cache: Some(false) }, + toolchain::TaskDefaults { + cache: Some(cacheable), + }, Some(uv_task_entrypoint(kind)), true, )) @@ -1161,6 +1186,7 @@ pub fn native_tasks_for_package( package: &str, package_directory: &str, workspace_directories: &[String], + build_cacheable: bool, ) -> Vec { let mut tasks = Vec::with_capacity(3); if kind == UvPackageKind::Package { @@ -1170,6 +1196,7 @@ pub fn native_tasks_for_package( vec!["build".to_string(), format!("--package={package}")], Vec::new(), None, + build_cacheable, )); } @@ -1192,14 +1219,23 @@ pub fn native_tasks_for_package( format_arguments, Vec::new(), None, + false, )); let check_arguments = match kind { UvPackageKind::Package | UvPackageKind::VirtualPackage => { - vec!["check".to_string(), format!("--package={package}")] + vec![ + "check".to_string(), + "--frozen".to_string(), + format!("--package={package}"), + ] } UvPackageKind::Workspace => { - vec!["check".to_string(), "--all-packages".to_string()] + vec![ + "check".to_string(), + "--frozen".to_string(), + "--all-packages".to_string(), + ] } }; tasks.push(uv_command_task( @@ -1208,6 +1244,7 @@ pub fn native_tasks_for_package( check_arguments, Vec::new(), Some("uv".to_string()), + false, )); if kind != UvPackageKind::Package { @@ -1238,6 +1275,7 @@ fn declared_tool_task( package: &str, targets: &[String], serial_group: Option, + toolchain_identified: bool, ) -> crate::native_tasks::NativeTask { let mut prefix = vec!["run".to_string(), "--frozen".to_string()]; match execution.owner { @@ -1267,7 +1305,14 @@ fn declared_tool_task( PythonTool::Ty => prefix.push("check".to_string()), PythonTool::Black | PythonTool::Mypy | PythonTool::Pyright | PythonTool::Pytest => {} } - uv_command_task(kind, task, prefix, targets.to_vec(), serial_group) + uv_command_task( + kind, + task, + prefix, + targets.to_vec(), + serial_group, + toolchain_identified && !task.starts_with("format"), + ) } fn pytest_task( @@ -1275,6 +1320,7 @@ fn pytest_task( execution: &ToolExecution, package: &str, package_directory: &str, + toolchain_identified: bool, ) -> crate::native_tasks::NativeTask { let targets = match kind { UvPackageKind::Package | UvPackageKind::VirtualPackage => { @@ -1290,6 +1336,7 @@ fn pytest_task( package, &targets, None, + toolchain_identified, ) } @@ -1324,6 +1371,8 @@ fn python_tasks_for_package( plan: &QualityPlan, pytest: Option<&ToolExecution>, emit_formatter_warning: bool, + toolchain_identified: bool, + build_cacheable: bool, ) -> Vec { let targets = match kind { UvPackageKind::Package | UvPackageKind::VirtualPackage => { @@ -1331,8 +1380,13 @@ fn python_tasks_for_package( } UvPackageKind::Workspace => workspace_directories.to_vec(), }; - let mut tasks = - native_tasks_for_package(kind, package, package_directory, workspace_directories); + let mut tasks = native_tasks_for_package( + kind, + package, + package_directory, + workspace_directories, + build_cacheable, + ); if plan.lint_homogeneous { let children: Vec<_> = plan @@ -1348,6 +1402,7 @@ fn python_tasks_for_package( package, &targets, Some("uv".to_string()), + toolchain_identified, )); name }) @@ -1374,6 +1429,7 @@ fn python_tasks_for_package( package, &targets, Some("uv".to_string()), + toolchain_identified, )); } if emit_formatter_warning { @@ -1387,6 +1443,7 @@ fn python_tasks_for_package( package, &targets, Some("uv".to_string()), + toolchain_identified, )); } } else { @@ -1407,6 +1464,7 @@ fn python_tasks_for_package( package, &targets, Some("uv".to_string()), + toolchain_identified, )); name }) @@ -1420,7 +1478,13 @@ fn python_tasks_for_package( } if let Some(execution) = pytest { - tasks.push(pytest_task(kind, execution, package, package_directory)); + tasks.push(pytest_task( + kind, + execution, + package, + package_directory, + toolchain_identified, + )); } const CLASSIFIED_TASKS: &[&str] = &[ @@ -1460,6 +1524,7 @@ pub const HASHED_ENV_VARS: &[&str] = &[ "UV_DEFAULT_INDEX", "UV_EXCLUDE", "UV_EXCLUDE_NEWER", + "UV_ENV_FILE", "UV_INDEX", "UV_INDEX_STRATEGY", "UV_INDEX_URL", @@ -1482,32 +1547,54 @@ pub const HASHED_ENV_VARS: &[&str] = &[ "UV_NO_BUILD", "UV_NO_BUILD_PACKAGE", "UV_NO_CONFIG", + "UV_NO_DEFAULT_GROUPS", + "UV_NO_DEV", "UV_NO_EDITABLE", + "UV_NO_ENV_FILE", "UV_NO_MANAGED_PYTHON", + "UV_NO_PROJECT", + "UV_NO_GROUP", "UV_NO_SOURCES_PACKAGE", "UV_NO_SYSTEM_CONFIG", "UV_NO_SOURCES", + "UV_NO_SYNC", "UV_OFFLINE", "UV_OVERRIDE", "UV_RESOLUTION", "UV_PRERELEASE", "UV_SYSTEM_CERTS", + "UV_ISOLATED", "UV_WORKING_DIR", "XDG_CONFIG_HOME", "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", + "PYTHONHOME", + "PYTHONPATH", ]; const UV_PATH_ENV_VARS: &[&str] = &[ "UV_BUILD_CONSTRAINT", "UV_CONFIG_FILE", "UV_CONSTRAINT", + "UV_ENV_FILE", "UV_EXCLUDE", "UV_OVERRIDE", "UV_PROJECT", + "UV_PROJECT_ENVIRONMENT", "UV_WORKING_DIR", + "PYTHONHOME", + "PYTHONPATH", ]; +fn environment_flag(environment: &toolchain::TaskIOEnvironment, name: &str) -> bool { + environment.get(name).is_some_and(|value| { + !matches!( + value.to_ascii_lowercase().as_str(), + "" | "0" | "false" | "no" + ) + }) +} + fn has_untracked_uv_path_env(environment: &toolchain::TaskIOEnvironment) -> bool { UV_PATH_ENV_VARS .iter() @@ -1518,12 +1605,13 @@ fn has_untracked_uv_configuration(environment: &toolchain::TaskIOEnvironment) -> if has_untracked_uv_path_env(environment) { return true; } - if environment.get("UV_NO_CONFIG").is_some_and(|value| { - !matches!( - value.to_ascii_lowercase().as_str(), - "" | "0" | "false" | "no" - ) - }) { + if environment_flag(environment, "UV_NO_SYNC") { + return true; + } + if environment_flag(environment, "UV_NO_PROJECT") { + return true; + } + if environment_flag(environment, "UV_NO_CONFIG") { return false; } let mut paths = Vec::new(); @@ -1666,6 +1754,11 @@ impl UvTaskContract { if has_untracked_uv_configuration(context.environment) { io.input_safety = toolchain::DerivedInputSafety::Untracked; } + if context.task_args.is_some_and(|args| !args.is_empty()) { + // Native tools accept path-valued and mutating options that cannot + // be inferred uniformly. Explicit cache configuration can opt in. + io.input_safety = toolchain::DerivedInputSafety::Untracked; + } match self.kind { UvPackageKind::Package | UvPackageKind::VirtualPackage => { if wants_automatic_inputs { @@ -1749,6 +1842,203 @@ impl UvTaskContract { // External dependency hashing // --------------------------------------------------------------------------- +#[derive(Debug, Deserialize, Serialize)] +struct UvPythonIdentity { + key: String, + version: String, + os: String, + variant: String, + implementation: String, + arch: String, + libc: String, + #[serde(default)] + binary_sha256: String, + #[serde(default)] + host: String, +} + +struct UvToolchainIdentity { + packages: [turborepo_lockfiles::Package; 2], + uv_version: node_semver::Version, +} + +fn bundled_uv_build_matches(requirement: &str, uv_version: &node_semver::Version) -> bool { + let Some(name) = pep508_name(requirement) else { + return false; + }; + if normalize_name(name) != "uv-build" { + return false; + } + let specifier = requirement[name.len()..].trim(); + if specifier.is_empty() { + return true; + } + let mut range = Vec::new(); + for clause in specifier.split(',').map(str::trim) { + let Some((operator, version)) = + [">=", "<=", "==", ">", "<"] + .into_iter() + .find_map(|operator| { + clause + .strip_prefix(operator) + .map(|version| (operator, version)) + }) + else { + return false; + }; + if version.is_empty() + || !version + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'.') + { + return false; + } + let mut release = version.split('.').collect::>(); + if release.len() > 3 || release.iter().any(|component| component.is_empty()) { + return false; + } + release.resize(3, "0"); + let version = release.join("."); + range.push(format!( + "{}{version}", + if operator == "==" { "=" } else { operator } + )); + } + node_semver::Range::parse(range.join(" ")).is_ok_and(|range| range.satisfies(uv_version)) +} + +fn parse_python_identity(stdout: &str, binary_sha256: String, host: String) -> Option { + let [mut identity]: [UvPythonIdentity; 1] = serde_json::from_str::>(stdout) + .ok()? + .try_into() + .ok()?; + identity.binary_sha256 = binary_sha256; + identity.host = host; + serde_json::to_string(&identity).ok() +} + +fn file_sha256(path: &std::path::Path) -> Option { + let mut file = std::fs::File::open(path).ok()?; + let mut hasher = Sha256::new(); + let mut buffer = [0; 64 * 1024]; + loop { + let read = file.read(&mut buffer).ok()?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Some(format!("{:x}", hasher.finalize())) +} + +#[cfg(target_os = "linux")] +fn host_compatibility_identity() -> Option { + let kernel = std::fs::read_to_string("/proc/sys/kernel/osrelease").ok()?; + let os_release = std::fs::read_to_string("/etc/os-release").unwrap_or_default(); + let runtime = ["/usr/bin/ldd", "/bin/ldd"].into_iter().find_map(|path| { + let path = std::path::Path::new(path); + path.exists().then(|| { + let output = Command::new(path).arg("--version").output().ok()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let value = format!("{stdout}{stderr}"); + (!value.trim().is_empty()).then(|| value.trim().to_string()) + })? + })?; + Some(format!("{kernel}\n{os_release}\n{runtime}")) +} + +#[cfg(target_os = "macos")] +fn host_compatibility_identity() -> Option { + let mut command = Command::new("/usr/bin/sw_vers"); + command.arg("-productVersion"); + successful_stdout(command) +} + +#[cfg(windows)] +fn host_compatibility_identity() -> Option { + let cmd = std::path::PathBuf::from(std::env::var_os("SystemRoot")?).join("System32/cmd.exe"); + let mut command = Command::new(cmd); + command.args(["/C", "ver"]); + successful_stdout(command) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +fn host_compatibility_identity() -> Option { + None +} + +fn successful_stdout(mut command: Command) -> Option { + let output = command.output().ok()?; + if !output.status.success() { + return None; + } + let stdout = std::str::from_utf8(&output.stdout).ok()?.trim(); + (!stdout.is_empty()).then(|| stdout.to_string()) +} + +/// Resolve the exact uv frontend and Python interpreter selected for this +/// workspace. Discovery remains available without either binary, but native +/// command tasks stay uncached until both identities can be proven. +fn toolchain_identities(repo_root: &AbsoluteSystemPath) -> Option { + let uv = std::fs::canonicalize(which::which("uv").ok()?).ok()?; + if uv.starts_with(repo_root.as_std_path()) { + return None; + } + let uv_sha256 = file_sha256(&uv)?; + + let mut uv_version = Command::new(&uv); + uv_version + .arg("--version") + .current_dir(repo_root.as_std_path()); + let uv_version_output = successful_stdout(uv_version)?; + let uv_version = node_semver::Version::parse( + uv_version_output + .strip_prefix("uv ")? + .split_whitespace() + .next()?, + ) + .ok()?; + let uv_identity = format!("{uv_version_output}\nsha256:{uv_sha256}"); + + let mut python = Command::new(&uv); + python + .args(["python", "find", "--resolve-links", "--no-python-downloads"]) + .current_dir(repo_root.as_std_path()); + let python = successful_stdout(python)?; + let python_path = std::fs::canonicalize(&python).ok()?; + let python_sha256 = file_sha256(&python_path)?; + let host = host_compatibility_identity()?; + + let mut python_identity = Command::new(&uv); + python_identity + .args([ + "python", + "list", + "--only-installed", + "--output-format", + "json", + python_path.to_str()?, + ]) + .current_dir(repo_root.as_std_path()); + let python_identity = successful_stdout(python_identity)?; + let python_identity = parse_python_identity(&python_identity, python_sha256, host)?; + + Some(UvToolchainIdentity { + packages: [ + turborepo_lockfiles::Package { + key: "uv".to_string(), + version: uv_identity, + }, + turborepo_lockfiles::Package { + key: "python".to_string(), + version: python_identity, + }, + ], + uv_version, + }) +} + /// Per-package external dependency closures from uv.lock, for the packages' /// external-dependency hashes. /// @@ -1961,6 +2251,8 @@ fn uv_change_observation(package_directories: &[String]) -> ChangeObservation { let mut observation = ChangeObservation::new() .with_rediscovery_file_name(PYPROJECT_TOML) .with_resolution_path(UV_LOCK) + .with_resolution_path(".python-version") + .with_resolution_path("uv.toml") .with_ignore_prefix(".venv") .with_ignore_prefix("dist"); for directory in std::iter::once("").chain(package_directories.iter().map(String::as_str)) { @@ -2080,11 +2372,22 @@ impl RepositoryContributor for UvContributor { ) .map_err(Error::from) .map_err(|err| toolchain::Error::Failed(Box::new(err)))?; + let toolchain_identity = + turborepo_rayon_compat::block_in_place(|| toolchain_identities(&self.repo_root)); + let toolchain_identified = toolchain_identity.is_some(); + let toolchain_packages = toolchain_identity + .as_ref() + .map(|identity| identity.packages.as_slice()) + .unwrap_or_default(); // The workspace-scoped closure covers every member plus the root // project's own dependencies (when the root is a package). - let workspace_externals: HashSet = - closures.values().flatten().cloned().collect(); + let workspace_externals: HashSet = closures + .values() + .flatten() + .cloned() + .chain(toolchain_packages.iter().cloned()) + .collect(); let mut discovered = Vec::with_capacity(packages.len() + 1); let mut resolutions = Vec::with_capacity(packages.len() + 1); @@ -2098,6 +2401,14 @@ impl RepositoryContributor for UvContributor { let package_directory = package_directories .get(&package.name) .map_or(".", String::as_str); + let build_cacheable = toolchain_identity.as_ref().is_some_and(|identity| { + package + .bundled_uv_build_requirement + .as_deref() + .is_some_and(|requirement| { + bundled_uv_build_matches(requirement, &identity.uv_version) + }) + }); let native_tasks = python_tasks_for_package( kind, &package.name, @@ -2106,6 +2417,8 @@ impl RepositoryContributor for UvContributor { &package.quality_plan, package.pytest.as_ref(), !workspace.quality_plan.format_homogeneous, + toolchain_identified, + build_cacheable, ); let task_contract = UvTaskContract::new(kind, &package.name); let mut external_dependencies = closures.remove(&package.name).unwrap_or_default(); @@ -2113,6 +2426,7 @@ impl RepositoryContributor for UvContributor { // Root-owned tools execute against the root environment. external_dependencies.extend(workspace_externals.iter().cloned()); } + external_dependencies.extend(toolchain_packages.iter().cloned()); resolutions.push(package_resolution( package.name.clone(), &external_dependencies, @@ -2144,6 +2458,8 @@ impl RepositoryContributor for UvContributor { &workspace.quality_plan, workspace.pytest.as_ref(), true, + toolchain_identified, + false, ); let workspace_task_contract = UvTaskContract::workspace(&workspace_name, workspace_directories); @@ -2761,6 +3077,31 @@ version = "0.1.0" ) .unwrap(); assert!(explicit_build.is_buildable()); + + let bundled: PyProjectManifest = toml::from_str( + "[project]\nname = \"app\"\n[build-system]\nrequires = \ + [\"uv_build>=0.12,<0.13\"]\nbuild-backend = \"uv_build\"\n", + ) + .unwrap(); + assert_eq!( + bundled.bundled_uv_build_requirement(), + Some("uv_build>=0.12,<0.13") + ); + } + + #[test] + fn test_bundled_uv_build_version_compatibility() { + let version = node_semver::Version::parse("0.12.1").unwrap(); + assert!(bundled_uv_build_matches("uv_build>=0.12,<0.13", &version)); + assert!(bundled_uv_build_matches("uv-build==0.12.1", &version)); + assert!(!bundled_uv_build_matches("uv-build==0.12", &version)); + assert!(bundled_uv_build_matches( + "uv-build==0.12", + &node_semver::Version::parse("0.12.0").unwrap() + )); + assert!(!bundled_uv_build_matches("uv_build>=0.13", &version)); + assert!(!bundled_uv_build_matches("uv_build~=0.12", &version)); + assert!(!bundled_uv_build_matches("hatchling>=1", &version)); } #[test] @@ -2952,6 +3293,28 @@ overridden = { index = "private" } assert!(!has_untracked_uv_configuration( &toolchain::TaskIOEnvironment::default() )); + + let no_sync = toolchain::TaskIOEnvironment::new(HashMap::from([( + "UV_NO_SYNC".to_string(), + "true".to_string(), + )])); + assert!(has_untracked_uv_configuration(&no_sync)); + } + + #[test] + fn test_python_identity_omits_installation_paths() { + let identity = parse_python_identity( + r#"[{"key":"cpython-3.13.11-linux-x86_64-gnu","version":"3.13.11","path":"/home/user/.local/python","symlink":null,"url":null,"os":"linux","variant":"default","implementation":"cpython","arch":"x86_64","libc":"gnu"}]"#, + "binary-hash".to_string(), + "host-identity".to_string(), + ) + .unwrap(); + + assert!(identity.contains("cpython-3.13.11-linux-x86_64-gnu")); + assert!(identity.contains("\"libc\":\"gnu\"")); + assert!(identity.contains("\"binary_sha256\":\"binary-hash\"")); + assert!(identity.contains("\"host\":\"host-identity\"")); + assert!(!identity.contains("/home/user")); } #[test] @@ -3054,6 +3417,8 @@ overridden = { index = "private" } &QualityPlan::effective(&ToolDeclarations::default(), &ToolDeclarations::default()), None, true, + true, + false, ); let display = |name| { tasks @@ -3063,9 +3428,13 @@ overridden = { index = "private" } }; assert_eq!(display("build"), Some("uv build --package=py-app")); assert_eq!(display("format"), Some("uv format -- packages/py-app")); - assert_eq!(display("check"), Some("uv check --package=py-app")); + assert_eq!(display("check"), Some("uv check --frozen --package=py-app")); let build = tasks.iter().find(|task| task.name() == "build").unwrap(); assert_eq!(build.contract().defaults().cache, Some(false)); + let format = tasks.iter().find(|task| task.name() == "format").unwrap(); + assert_eq!(format.contract().defaults().cache, Some(false)); + let check = tasks.iter().find(|task| task.name() == "check").unwrap(); + assert_eq!(check.contract().defaults().cache, Some(false)); assert_eq!( build.contract().entrypoint(), Some(crate::native_tasks::TaskEntrypoint::Candidate) @@ -3097,6 +3466,8 @@ overridden = { index = "private" } &QualityPlan::default(), Some(&root_execution), true, + true, + false, ); let root_test = root_tasks .iter() @@ -3123,6 +3494,8 @@ overridden = { index = "private" } &QualityPlan::default(), Some(&member_execution), true, + true, + false, ); let member_test = member_tasks .iter() @@ -3179,6 +3552,8 @@ overridden = { index = "private" } &plan, None, true, + true, + false, ); let task = |name| tasks.iter().find(|task| task.name() == name).unwrap(); assert_eq!( @@ -3235,6 +3610,8 @@ overridden = { index = "private" } &plan, None, true, + true, + false, ); let lint = tasks .iter() @@ -3276,6 +3653,8 @@ overridden = { index = "private" } &plan, None, true, + true, + false, ); let task = |name| tasks.iter().find(|task| task.name() == name).unwrap(); assert_eq!( @@ -3324,7 +3703,7 @@ overridden = { index = "private" } let mypy = task("check:mypy"); assert_eq!(mypy.command().unwrap().serial_group.as_deref(), Some("uv")); - assert_eq!(mypy.contract().defaults().cache, Some(false)); + assert_eq!(mypy.contract().defaults().cache, Some(true)); assert_eq!( mypy.contract().entrypoint(), Some(crate::native_tasks::TaskEntrypoint::Candidate) @@ -3332,6 +3711,26 @@ overridden = { index = "private" } assert!(mypy.contract().derives_io()); } + #[test] + fn test_uv_commands_stay_uncached_without_toolchain_identity() { + let tasks = native_tasks_for_package( + UvPackageKind::Package, + "py-app", + "packages/py-app", + &[], + false, + ); + + for task in tasks.iter().filter(|task| task.command().is_some()) { + assert_eq!( + task.contract().defaults().cache, + Some(false), + "{} must fail closed", + task.name() + ); + } + } + #[test] fn test_derived_outputs_for_build() { let contract = UvTaskContract::new(UvPackageKind::Package, "py-app"); @@ -3411,6 +3810,16 @@ overridden = { index = "private" } ); assert!(io.input_globs.contains(&"!**/__pycache__/**".to_string())); assert!(io.input_globs.contains(&"!.pytest_cache/**".to_string())); + + let args = vec!["--fix".to_string()]; + let context = toolchain::TaskIOContext { + task_args: Some(&args), + environment: &environment, + }; + let io = UvTaskContract::new(UvPackageKind::VirtualPackage, "app") + .derived_task_io(&package, "check", "../..", &[], true, &context) + .unwrap(); + assert_eq!(io.input_safety, toolchain::DerivedInputSafety::Untracked); } #[test] @@ -3470,6 +3879,8 @@ overridden = { index = "private" } let expected = ChangeObservation::new() .with_rediscovery_file_name(PYPROJECT_TOML) .with_resolution_path(UV_LOCK) + .with_resolution_path(".python-version") + .with_resolution_path("uv.toml") .with_ignore_prefix(".venv") .with_ignore_prefix("dist"); let expected = ["", "packages/app"] diff --git a/crates/turborepo/ARCHITECTURE.md b/crates/turborepo/ARCHITECTURE.md index 3e0765fd54ece..fe1e57585af0f 100644 --- a/crates/turborepo/ARCHITECTURE.md +++ b/crates/turborepo/ARCHITECTURE.md @@ -657,8 +657,11 @@ tasks, external resolution, change observations, and a prune domain through the shared repository graph. - **Discovery** parses `[tool.uv.workspace] members` and `exclude` globs - in-process, so graph construction does not require the `uv` binary. Names - are PEP 503-normalized. Dependencies become internal graph edges only when + in-process, so graph construction does not require the `uv` binary. When uv + is available, discovery probes its version and the Python interpreter selected + by `uv python find` for cache identity. Missing identities and repository-local + uv executables fail closed to uncached tasks. Names are PEP 503-normalized. + Dependencies become internal graph edges only when their effective `[tool.uv.sources]` entry selects `workspace = true`. Development edges that would create a cycle become non-ordering input edges. A root `[project]` participates in hashing and pruning but is not a @@ -700,12 +703,18 @@ the shared repository graph. --group `), the tool and subcommand, pass-through arguments, and the member-directory targets. Ruff uses `check`/`format`; ty uses `check`. Fallbacks remain `uv format -- ` and either - `uv check --package=` or `uv check --all-packages`. Detected-tool and - fallback `check` commands use the `uv` serial group. All command tasks default - to uncached. Detected-tool commands use `--frozen`; Turborepo itself never - creates or updates `uv.lock`. Pass-through arguments are inserted before path - targets. Active aggregates reject them and name the package-qualified child - tasks that can receive them. + `uv check --frozen --package=` or `uv check --frozen --all-packages`. + Detected-tool and + fallback `check` commands use the `uv` serial group. Detected lint, check, and + test commands default to cacheable when uv and Python identities resolve; + otherwise they fail closed to uncached. The fallback `uv check` stays uncached + because its bundled checker is not represented in `uv.lock`. Builds using + `uv_build` cache when their sole build requirement accepts the identified uv + executable's bundled backend version. Other PEP 517 builds remain uncached, + and format commands remain uncached because they mutate source. Detected-tool commands use `--frozen`; + Turborepo itself never creates or updates `uv.lock`. Pass-through arguments are + inserted before path targets. Active aggregates reject them and name the + package-qualified child tasks that can receive them. Pytest commands are `uv run --frozen pytest` for the workspace or `uv run --frozen --package pytest ` for members, with non-default group activation inserted before pytest and pass-through arguments inserted @@ -716,15 +725,18 @@ the shared repository graph. Quality workspace tasks include every member's sources; a bare workspace pytest task hashes the full repository because pytest controls collection. Quality caches, `.pytest_cache`, `.venv`, and `__pycache__` are excluded. - Path-valued uv settings and active user/system uv configuration - make automatic inputs untracked. Each scope also hashes its external + Path-valued uv settings, `UV_NO_SYNC`, `UV_NO_PROJECT`, active user/system uv + configuration, and any pass-through arguments make automatic inputs + untracked. Each scope also hashes its external dependency closure from `uv.lock`; root-owned tools conservatively add the workspace closure. Package identities include version, source, and artifact - hashes. A `uv.lock` change across git refs conservatively affects all uv - packages. Build output inference covers the bare command's matching `dist/` - artifacts and becomes unavailable when arguments are present. -- **Watch mode** rediscoveries follow any `pyproject.toml` and the root - `uv.lock`. Root `.venv/` and `dist/`, plus known quality-tool and Python cache + hashes. Every scope also includes path-independent uv and Python identities + containing executable content hashes, Python implementation and version, + operating system, architecture, libc, variant, and host compatibility. A `uv.lock` change across git refs conservatively affects + all uv packages. Build output inference covers the bare command's matching + `dist/` artifacts and becomes unavailable when arguments are present. +- **Watch mode** rediscoveries follow any `pyproject.toml`, the root `uv.lock`, + `.python-version`, and `uv.toml`. Root `.venv/` and `dist/`, plus known quality-tool and Python cache directories at the root and member scopes, are ignored as task byproducts. - **Prune** walks `uv.lock` reachability, including dependency groups and optional extras, and preserves retained package metadata through diff --git a/crates/turborepo/tests/uv_workspace_test.rs b/crates/turborepo/tests/uv_workspace_test.rs index 189208330b1ef..1a9d3a363d24e 100644 --- a/crates/turborepo/tests/uv_workspace_test.rs +++ b/crates/turborepo/tests/uv_workspace_test.rs @@ -132,7 +132,7 @@ fn test_pure_uv_workspace_task_graph() { assert_eq!(task_ids(&json), vec!["acme#check".to_string()]); assert_eq!( find_task(&json, "acme#check")["command"], - "uv check --all-packages" + "uv check --frozen --all-packages" ); } @@ -158,7 +158,7 @@ fn test_uv_filter_by_package() { assert_eq!(task_ids(&json), vec!["py-app#check".to_string()]); assert_eq!( find_task(&json, "py-app#check")["command"], - "uv check --package=py-app" + "uv check --frozen --package=py-app" ); let json = dry_run_tasks(tempdir.path(), &["build"]); @@ -210,10 +210,6 @@ dev = ["Ruff", "black", "mypy", "ty", "pyright"] find_task(&check, "acme#check:mypy")["command"], "uv run --frozen mypy packages/py-app packages/py-lib" ); - assert_eq!( - find_task(&check, "acme#check:mypy")["resolvedTaskDefinition"]["cache"], - false - ); let format = dry_run_tasks(tempdir.path(), &["format"]); assert_eq!(task_ids(&format), vec!["acme#format".to_string()]); @@ -556,23 +552,32 @@ fn test_uv_lock_change_affects_all_packages() { } #[test] -fn test_uv_build_executes_without_caching() { +fn test_uv_build_caches_bundled_backend() { if !uv_available() { return; } let tempdir = tempfile::tempdir().unwrap(); setup_uv_pure_workspace(tempdir.path()); - let dry_run = dry_run_tasks(tempdir.path(), &["build", "--filter=py-app"]); + let run = |args: &[&str]| { + let config_dir = tempfile::tempdir().expect("failed to create config tempdir"); + let mut command = common::turbo_command(tempdir.path()); + command + .env("TURBO_CONFIG_DIR_PATH", config_dir.path()) + .env("UV_NO_CONFIG", "1") + .args(args) + .output() + .expect("failed to execute turbo") + }; + let output = run(&["build", "--filter=py-app", "--dry-run=json"]); + assert_command_success(&output, "build dry-run"); + let dry_run: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); assert_eq!( find_task(&dry_run, "py-app#build")["resolvedTaskDefinition"]["cache"], - false + true ); - let output = run_turbo( - tempdir.path(), - &["build", "--filter=py-app", "--log-order", "grouped"], - ); + let output = run(&["build", "--filter=py-app", "--log-order", "grouped"]); assert_command_success(&output, "first build"); let wheel_exists = || { fs::read_dir(tempdir.path().join("dist")) @@ -585,10 +590,68 @@ fn test_uv_build_executes_without_caching() { }; assert!(wheel_exists(), "uv build must produce a wheel in dist/"); + fs::remove_dir_all(tempdir.path().join("dist")).unwrap(); + let output = run(&["build", "--filter=py-app", "--log-order", "grouped"]); + assert_command_success(&output, "cached build"); let stdout = String::from_utf8_lossy(&output.stdout); assert!( - !stdout.contains("FULL TURBO"), - "build must be uncached: {stdout}" + stdout.contains("FULL TURBO"), + "expected build cache hit: {stdout}" + ); + assert!(wheel_exists(), "cache hit must restore the wheel"); +} + +#[test] +fn test_uv_quality_tasks_cache_with_toolchain_identity() { + if !uv_available() { + return; + } + let tempdir = tempfile::tempdir().unwrap(); + setup_uv_pure_workspace(tempdir.path()); + append_manifest( + tempdir.path(), + "pyproject.toml", + "\n[dependency-groups]\ndev = [\"ruff\"]\n", + ); + let lock = std::process::Command::new("uv") + .arg("lock") + .current_dir(tempdir.path()) + .output() + .expect("uv lock runs"); + assert_command_success(&lock, "uv lock"); + + let config_dir = tempfile::tempdir().expect("failed to create config tempdir"); + let mut command = common::turbo_command(tempdir.path()); + let output = command + .env("TURBO_CONFIG_DIR_PATH", config_dir.path()) + .env("UV_NO_CONFIG", "1") + .args(["lint:ruff", "--filter=py-lib", "--dry-run=json"]) + .output() + .expect("failed to execute turbo"); + assert_command_success(&output, "cacheable quality dry-run"); + let dry_run: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + find_task(&dry_run, "py-lib#lint:ruff")["resolvedTaskDefinition"]["cache"], + true + ); + + let run = || { + let config_dir = tempfile::tempdir().expect("failed to create config tempdir"); + let mut command = common::turbo_command(tempdir.path()); + command + .env("TURBO_CONFIG_DIR_PATH", config_dir.path()) + .env("UV_NO_CONFIG", "1") + .args(["lint:ruff", "--filter=py-lib", "--log-order=grouped"]) + .output() + .expect("failed to execute turbo") + }; + assert_command_success(&run(), "first cacheable quality run"); + let second = run(); + assert_command_success(&second, "second cacheable quality run"); + let stdout = String::from_utf8_lossy(&second.stdout); + assert!( + stdout.contains("FULL TURBO"), + "expected cache hit: {stdout}" ); }