Skip to content

fix(automation): poll artifact_glob after wait_for_idle settles (fixes #120) #121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bketelsen wants to merge 5 commits into from
Closed

fix(automation): poll artifact_glob after wait_for_idle settles (fixes #120) #121

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5884891
fix(automation): poll artifact_glob after wait_for_idle settles
bketelsen May 2, 2026
24e0424
fix: reuse artifact polling on implement_code exits
nutt-adam May 3, 2026
770d9c0
fix: fail required artifact capture on early exits
nutt-adam May 4, 2026
626aba5
Merge branch 'main' into fix/artifact-capture-poll-after-idle
nutt-adam May 4, 2026
cde9b4e
Merge branch 'main' into fix/artifact-capture-poll-after-idle
nutt-adam May 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 200 additions & 35 deletions src/automation/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1035,6 +1035,7 @@ impl<'a> WorkflowExecutor<'a> {
} else {
None
};
let post_idle_poll_secs = post_idle_artifact_poll_secs(*step_wait_timeout);

let baseline_pane_hash =
TmuxSession::capture_pane(session_name, PROMPT_CAPTURE_LINES)
Expand Down Expand Up @@ -1564,22 +1565,65 @@ impl<'a> WorkflowExecutor<'a> {
if let (Some((expanded_pattern, pre_snap)), Some(art_name)) =
(artifact_pre_snapshot.as_ref(), artifact_name.as_deref())
{
std::thread::sleep(Duration::from_secs(2));
if let Ok(artifact_path) =
capture_artifact(pre_snap, expanded_pattern, art_name)
&& let Ok(result) = store_artifact_output(
self.project_root,
&run_id,
art_name,
&artifact_path,
)
{
output_files.insert(
art_name.to_string(),
result.json_path.display().to_string(),
);
outputs.insert(art_name.to_string(), result.value);
}
let artifact_path = match capture_artifact_with_poll(
pre_snap,
expanded_pattern,
art_name,
post_idle_poll_secs,
Duration::from_secs(1),
) {
Ok(path) => path,
Err(e) => {
failed_steps.push(step_index);
success = false;
step_results.push(StepResult {
index: step_index,
step_type: "prompt".to_string(),
status: StepStatus::Failed,
duration_ms: started.elapsed().as_millis() as u64,
exit_code: None,
timed_out: false,
message: Some(e.to_string()),
stdout: None,
stderr: None,
});
break;
}
};

let result = match store_artifact_output(
self.project_root,
&run_id,
art_name,
&artifact_path,
) {
Ok(result) => result,
Err(e) => {
failed_steps.push(step_index);
success = false;
step_results.push(StepResult {
index: step_index,
step_type: "prompt".to_string(),
status: StepStatus::Failed,
duration_ms: started.elapsed().as_millis() as u64,
exit_code: None,
timed_out: false,
message: Some(format!(
"artifact capture failed for '{}': {e}",
art_name
)),
stdout: None,
stderr: None,
});
break;
}
};

output_files.insert(
art_name.to_string(),
result.json_path.display().to_string(),
);
outputs.insert(art_name.to_string(), result.value);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
step_results.push(StepResult {
index: step_index,
Expand Down Expand Up @@ -1628,22 +1672,67 @@ impl<'a> WorkflowExecutor<'a> {
if let (Some((expanded_pattern, pre_snap)), Some(art_name)) =
(artifact_pre_snapshot.as_ref(), artifact_name.as_deref())
{
std::thread::sleep(Duration::from_secs(2));
if let Ok(artifact_path) =
capture_artifact(pre_snap, expanded_pattern, art_name)
&& let Ok(result) = store_artifact_output(
self.project_root,
&run_id,
art_name,
&artifact_path,
)
{
output_files.insert(
art_name.to_string(),
result.json_path.display().to_string(),
);
outputs.insert(art_name.to_string(), result.value);
}
let artifact_path = match capture_artifact_with_poll(
pre_snap,
expanded_pattern,
art_name,
post_idle_poll_secs,
Duration::from_secs(1),
) {
Ok(path) => path,
Err(e) => {
failed_steps.push(step_index);
success = false;
step_results.push(StepResult {
index: step_index,
step_type: "prompt".to_string(),
status: StepStatus::Failed,
duration_ms: started.elapsed().as_millis()
as u64,
exit_code: None,
timed_out: false,
message: Some(e.to_string()),
stdout: None,
stderr: None,
});
break;
}
};

let result = match store_artifact_output(
self.project_root,
&run_id,
art_name,
&artifact_path,
) {
Ok(result) => result,
Err(e) => {
failed_steps.push(step_index);
success = false;
step_results.push(StepResult {
index: step_index,
step_type: "prompt".to_string(),
status: StepStatus::Failed,
duration_ms: started.elapsed().as_millis()
as u64,
exit_code: None,
timed_out: false,
message: Some(format!(
"artifact capture failed for '{}': {e}",
art_name
)),
stdout: None,
stderr: None,
});
break;
}
};

output_files.insert(
art_name.to_string(),
result.json_path.display().to_string(),
);
outputs.insert(art_name.to_string(), result.value);
}
step_results.push(StepResult {
index: step_index,
Expand Down Expand Up @@ -1714,10 +1803,22 @@ impl<'a> WorkflowExecutor<'a> {
if let (Some((expanded_pattern, pre_snap)), Some(art_name)) =
(artifact_pre_snapshot.as_ref(), artifact_name.as_deref())
{
// Brief settle time for filesystem I/O
std::thread::sleep(Duration::from_secs(2));
// Poll for the artifact to materialize. wait_for_agent_idle
// can return as soon as the runtime emits a completion signal,
// which may be before the agent's file write is flushed to disk.
// Use a short bounded poll window proportional to wait_timeout_secs
// (capped at 60s) so that fast steps stay fast and slow flushes
// don't false-fail.
// See https://github.com/nutthouse/tutti/issues/120
let capture_result = capture_artifact_with_poll(
pre_snap,
expanded_pattern,
art_name,
post_idle_poll_secs,
Duration::from_secs(1),
);

match capture_artifact(pre_snap, expanded_pattern, art_name) {
match capture_result {
Ok(artifact_path) => {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
match store_artifact_output(
self.project_root,
Expand Down Expand Up @@ -2779,6 +2880,33 @@ fn snapshot_artifact_glob(pattern: &str) -> Result<HashSet<PathBuf>> {
Ok(paths.filter_map(|p| p.ok()).collect())
}

fn post_idle_artifact_poll_secs(wait_timeout_secs: u64) -> u64 {
(wait_timeout_secs / 30).clamp(5, 60)
}

fn capture_artifact_with_poll(
pre_snapshot: &HashSet<PathBuf>,
pattern: &str,
artifact_name: &str,
poll_secs: u64,
poll_interval: Duration,
) -> Result<PathBuf> {
let poll_deadline = Duration::from_secs(poll_secs);
let poll_start = std::time::Instant::now();

loop {
match capture_artifact(pre_snapshot, pattern, artifact_name) {
Ok(path) => break Ok(path),
Err(err) => {
if poll_start.elapsed() >= poll_deadline {
break Err(err);
}
std::thread::sleep(poll_interval);
}
}
}
}

/// After a step completes, capture the newest artifact that appeared since the pre-step snapshot.
fn capture_artifact(
pre_snapshot: &HashSet<PathBuf>,
Expand Down Expand Up @@ -6317,6 +6445,43 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn capture_artifact_with_poll_waits_for_late_file() {
let dir = std::env::temp_dir().join("tutti-test-capture-poll");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();

let pattern = format!("{}/*.md", dir.display());
let pre_snapshot = snapshot_artifact_glob(&pattern).unwrap();
let late_file = dir.join("design-late.md");
let writer_path = late_file.clone();

let writer = std::thread::spawn(move || {
std::thread::sleep(std::time::Duration::from_millis(25));
std::fs::write(&writer_path, "late artifact").unwrap();
});

let result = capture_artifact_with_poll(
&pre_snapshot,
&pattern,
"design_doc",
1,
std::time::Duration::from_millis(5),
)
.unwrap();
writer.join().unwrap();
assert_eq!(result, late_file);

let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn post_idle_artifact_poll_secs_is_bounded() {
assert_eq!(post_idle_artifact_poll_secs(30), 5);
assert_eq!(post_idle_artifact_poll_secs(1800), 60);
assert_eq!(post_idle_artifact_poll_secs(3600), 60);
}

#[test]
fn capture_artifact_multiple_new_files_picks_newest() {
let dir = std::env::temp_dir().join("tutti-test-capture-multi");
Expand Down