diff --git a/crates/turborepo-task-hash/src/lib.rs b/crates/turborepo-task-hash/src/lib.rs index 5a2fb8379d584..4bd70b6dee417 100644 --- a/crates/turborepo-task-hash/src/lib.rs +++ b/crates/turborepo-task-hash/src/lib.rs @@ -1101,6 +1101,8 @@ mod test { #[path = "dependency_outputs_test.rs"] mod dependency_outputs; + #[path = "jit_inputs_test.rs"] + mod jit_inputs; struct TestRunOpts { single_package: bool, diff --git a/crates/turborepo-task-hash/src/test/jit_inputs_test.rs b/crates/turborepo-task-hash/src/test/jit_inputs_test.rs new file mode 100644 index 0000000000000..7694f9038f5ae --- /dev/null +++ b/crates/turborepo-task-hash/src/test/jit_inputs_test.rs @@ -0,0 +1,253 @@ +use super::*; + +fn task_hasher_with_file_inputs<'a>( + repo_root: &'a AbsoluteSystemPathBuf, + run_opts: &'a TestRunOpts, + env: &'a EnvironmentVariableMap, + file_inputs: HashMap, Arc>, +) -> TaskHasher<'a, TestRunOpts> { + let hashes = file_inputs + .iter() + .map(|(task_id, file_hashes)| (task_id.clone(), file_hashes.as_ref().clone().hash())) + .collect(); + TaskHasher::new( + PackageInputsHashes { + hashes, + expanded_hashes: file_inputs, + }, + run_opts, + env, + "global-hash", + repo_root, + EnvironmentVariableMap::default(), + &[], + ) +} + +fn jit_task_definition(globs: &[&str]) -> TaskDefinition { + TaskDefinition { + inputs: TaskInputs { + jit_globs: globs.iter().map(|glob| glob.to_string()).collect(), + eager: false, + ..TaskInputs::default() + }, + ..TaskDefinition::default() + } +} + +fn calculate_deferred_task_hash( + hasher: &TaskHasher<'_, TestRunOpts>, + repo_root: &AbsoluteSystemPathBuf, + graph: &PackageGraph, + task_id: &TaskId<'static>, + definition: &TaskDefinition, +) -> String { + let package = PackageName::from(task_id.package()); + hasher + .calculate_task_hash_with_deferred_inputs( + task_id, + definition, + EnvMode::Strict, + &graph.package_task_context(&package).unwrap(), + &[], + PackageTaskEventBuilder::new(task_id.package(), task_id.task()), + &SCM::new(repo_root), + repo_root, + None, + None, + &HashSet::new(), + ) + .unwrap() +} + +fn write_file(repo_root: &AbsoluteSystemPathBuf, package: &str, path: &str, contents: &str) { + let mut components = vec!["packages", package]; + components.extend(path.split('/')); + let absolute_path = repo_root.join_components(&components); + absolute_path.ensure_dir().unwrap(); + absolute_path.create_with_contents(contents).unwrap(); +} + +fn empty_eager_inputs() -> Arc { + Arc::new(FileHashes(Vec::new())) +} + +#[tokio::test] +async fn jit_inputs_are_hashed_after_dependency_files_are_materialized() { + let tmp = tempdir().unwrap(); + let repo_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap(); + let graph = javascript_graph(&repo_root).await; + let task_id = TaskId::new("app", "build").into_owned(); + let definition = jit_task_definition(&["src/generated/**"]); + let run_opts = TestRunOpts { + single_package: true, + }; + let env = EnvironmentVariableMap::default(); + + write_file(&repo_root, "app", "src/generated/schema.txt", "schema-v1\n"); + let first_hasher = task_hasher_with_file_inputs( + &repo_root, + &run_opts, + &env, + HashMap::from([(task_id.clone(), empty_eager_inputs())]), + ); + first_hasher + .insert_deferred_hash( + &task_id, + &definition, + EnvMode::Strict, + &graph + .package_task_context(&PackageName::from("app")) + .unwrap(), + ) + .unwrap(); + assert_eq!( + first_hasher.task_hash_tracker().hash(&task_id).as_deref(), + Some(JIT_DEFERRED_TASK_HASH_MESSAGE) + ); + let first = + calculate_deferred_task_hash(&first_hasher, &repo_root, &graph, &task_id, &definition); + + // This file represents output produced by a dependency before the task's + // deferred JIT inputs are hashed. + write_file(&repo_root, "app", "src/generated/schema.txt", "schema-v2\n"); + let second_hasher = task_hasher_with_file_inputs( + &repo_root, + &run_opts, + &env, + HashMap::from([(task_id.clone(), empty_eager_inputs())]), + ); + let second = + calculate_deferred_task_hash(&second_hasher, &repo_root, &graph, &task_id, &definition); + + assert_ne!( + first, second, + "JIT output changes must change the task hash" + ); +} + +fn hash_jit_dependency_and_startup_descendant( + repo_root: &AbsoluteSystemPathBuf, + graph: &PackageGraph, + generate: &TaskId<'static>, + generate_definition: &TaskDefinition, + build: &TaskId<'static>, + build_definition: &TaskDefinition, + build_inputs: Arc, +) -> (String, String) { + let run_opts = TestRunOpts { + single_package: true, + }; + let env = EnvironmentVariableMap::default(); + let hasher = task_hasher_with_file_inputs( + repo_root, + &run_opts, + &env, + HashMap::from([ + (generate.clone(), empty_eager_inputs()), + (build.clone(), build_inputs), + ]), + ); + let generate_context = graph + .package_task_context(&PackageName::from(generate.package())) + .unwrap(); + hasher + .insert_deferred_hash( + generate, + generate_definition, + EnvMode::Strict, + &generate_context, + ) + .unwrap(); + let generate_hash = + calculate_deferred_task_hash(&hasher, repo_root, graph, generate, generate_definition); + + let build_context = graph + .package_task_context(&PackageName::from(build.package())) + .unwrap(); + let generate_node = TaskNode::Task(generate.clone()); + let build_hash = hasher + .calculate_task_hash( + build, + build_definition, + EnvMode::Strict, + &build_context, + &[&generate_node], + PackageTaskEventBuilder::new(build.package(), build.task()), + ) + .unwrap(); + (generate_hash, build_hash) +} + +#[tokio::test] +async fn descendants_hash_with_resolved_jit_dependency_and_pre_dependency_startup_inputs() { + let tmp = tempdir().unwrap(); + let repo_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap(); + let graph = javascript_graph(&repo_root).await; + let generate = TaskId::new("app", "generate").into_owned(); + let build = TaskId::new("app", "build").into_owned(); + let generate_definition = jit_task_definition(&["jit-input.txt"]); + let build_definition = TaskDefinition { + inputs: TaskInputs::new(vec!["marker.txt".to_string()]), + ..TaskDefinition::default() + }; + + write_file(&repo_root, "app", "marker.txt", "before\n"); + write_file(&repo_root, "app", "jit-input.txt", "stable\n"); + let build_context = graph + .package_task_context(&PackageName::from("app")) + .unwrap(); + let first_build_inputs = file_hashes_for_inputs( + &SCM::new(&repo_root), + &repo_root, + build_context.directory(), + &["marker.txt"], + false, + None, + ) + .unwrap(); + + // Simulate the dependency finishing after startup inputs were snapshotted. + write_file(&repo_root, "app", ".generated/done.txt", "done\n"); + write_file(&repo_root, "app", "marker.txt", "after\n"); + let first = hash_jit_dependency_and_startup_descendant( + &repo_root, + &graph, + &generate, + &generate_definition, + &build, + &build_definition, + first_build_inputs, + ); + + // A later run observes the same startup marker before the dependency + // executes. The dependency may rewrite it again, but the descendant's + // hash uses the startup snapshot and the resolved JIT dependency hash. + write_file(&repo_root, "app", "marker.txt", "before\n"); + let second_build_inputs = file_hashes_for_inputs( + &SCM::new(&repo_root), + &repo_root, + build_context.directory(), + &["marker.txt"], + false, + None, + ) + .unwrap(); + write_file(&repo_root, "app", ".generated/done.txt", "done\n"); + write_file(&repo_root, "app", "marker.txt", "after\n"); + let second = hash_jit_dependency_and_startup_descendant( + &repo_root, + &graph, + &generate, + &generate_definition, + &build, + &build_definition, + second_build_inputs, + ); + + assert_eq!(first.0, second.0, "stable JIT inputs produce a stable hash"); + assert_eq!( + first.1, second.1, + "the descendant uses the startup snapshot and resolved JIT hash" + ); +} diff --git a/crates/turborepo/tests/run_caching.rs b/crates/turborepo/tests/run_caching.rs index 3738437024345..3a7f18f3ab086 100644 --- a/crates/turborepo/tests/run_caching.rs +++ b/crates/turborepo/tests/run_caching.rs @@ -221,164 +221,6 @@ fn test_excluded_inputs() { assert!(stdout.contains("FULL TURBO")); } -#[test] -fn test_structured_jit_inputs_hash_after_dependencies_complete() { - let tempdir = tempfile::tempdir().unwrap(); - setup::setup_integration_test(tempdir.path(), "basic_monorepo", "npm@10.5.0", true).unwrap(); - - fs::write( - tempdir.path().join("apps/my-app/package.json"), - r#"{ - "name": "my-app", - "scripts": { - "codegen": "node -e \"require('fs').mkdirSync('src/generated', { recursive: true }); require('fs').writeFileSync('src/generated/schema.txt', 'schema-v1\\n')\"", - "build": "node -e \"const fs = require('fs'); fs.mkdirSync('.output', { recursive: true }); fs.writeFileSync('.output/result.txt', fs.readFileSync('src/generated/schema.txt'))\"" - }, - "dependencies": { - "util": "*" - } -} -"#, - ) - .unwrap(); - - fs::write( - tempdir.path().join("turbo.json"), - r#"{ - "$schema": "https://turborepo.dev/schema.json", - "tasks": { - "codegen": { - "inputs": ["$TURBO_DEFAULT$", "!src/generated/**", "!.output/**"], - "outputs": ["src/generated/**"] - }, - "build": { - "dependsOn": ["codegen"], - "inputs": [ - "$TURBO_DEFAULT$", - "!.output/**", - { - "mode": "jit", - "globs": ["src/generated/**"] - } - ], - "outputs": [".output/**"] - } - } -} -"#, - ) - .unwrap(); - - let output = run_turbo( - tempdir.path(), - &["run", "build", "--filter=my-app", "--dry=json"], - ); - let json: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); - let build_task = json["tasks"] - .as_array() - .unwrap() - .iter() - .find(|task| task["taskId"] == "my-app#build") - .unwrap(); - assert!(build_task["hash"].is_null()); - assert_eq!( - build_task["hashReason"], - "Deferred because JIT hashing mode was used." - ); - - let output = run_turbo( - tempdir.path(), - &["run", "build", "--filter=my-app", "--output-logs=none"], - ); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - stdout.contains("0 cached, 2 total"), - "expected first run to execute both tasks, got:\n{stdout}" - ); - - let output = run_turbo( - tempdir.path(), - &["run", "build", "--filter=my-app", "--output-logs=none"], - ); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - stdout.contains("2 cached, 2 total"), - "expected second run to cache both tasks after one run, got:\n{stdout}" - ); - assert!(stdout.contains("FULL TURBO")); -} - -#[test] -fn test_structured_jit_descendant_hashes_after_jit_hash_is_available() { - let tempdir = tempfile::tempdir().unwrap(); - setup::setup_integration_test(tempdir.path(), "basic_monorepo", "npm@10.5.0", true).unwrap(); - - fs::write(tempdir.path().join("apps/my-app/marker.txt"), "before\n").unwrap(); - fs::write(tempdir.path().join("apps/my-app/jit-input.txt"), "stable\n").unwrap(); - fs::write( - tempdir.path().join("apps/my-app/package.json"), - r#"{ - "name": "my-app", - "scripts": { - "generate": "node -e \"const fs = require('fs'); fs.mkdirSync('.generated', { recursive: true }); fs.writeFileSync('.generated/done.txt', 'done\\n'); fs.writeFileSync('marker.txt', 'after\\n')\"", - "build": "node -e \"const fs = require('fs'); fs.mkdirSync('.output', { recursive: true }); fs.writeFileSync('.output/marker.txt', fs.readFileSync('marker.txt'))\"" - }, - "dependencies": { - "util": "*" - } -} -"#, - ) - .unwrap(); - - fs::write( - tempdir.path().join("turbo.json"), - r#"{ - "$schema": "https://turborepo.dev/schema.json", - "tasks": { - "generate": { - "inputs": [ - { - "mode": "jit", - "globs": ["jit-input.txt"] - } - ], - "outputs": [".generated/**"] - }, - "build": { - "dependsOn": ["generate"], - "inputs": ["marker.txt"], - "outputs": [".output/**"] - } - } -} -"#, - ) - .unwrap(); - - let output = run_turbo( - tempdir.path(), - &["run", "build", "--filter=my-app", "--output-logs=none"], - ); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - stdout.contains("0 cached, 2 total"), - "expected first run to execute both tasks, got:\n{stdout}" - ); - - fs::write(tempdir.path().join("apps/my-app/marker.txt"), "before\n").unwrap(); - - let output = run_turbo( - tempdir.path(), - &["run", "build", "--filter=my-app", "--output-logs=none"], - ); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - stdout.contains("2 cached, 2 total"), - "expected JIT descendants to be hashed before the dependency command runs, got:\n{stdout}" - ); -} - #[test] fn test_structured_startup_with_defaults_matches_legacy_startup_semantics() { let tempdir = tempfile::tempdir().unwrap();