test: add 29 E2E trace tests for issues #571-575

tests/e2e_builtin_tool_coverage.rs

@@ -0,0 +1,332 @@
+//! E2E trace tests: builtin tool coverage (#573).
+//!
+//! Covers time (parse, diff, invalid), routine (create, list, update, delete,
+//! history), job (create, status, list, cancel), and HTTP replay.
+
+#[cfg(feature = "libsql")]
+mod support;
+
+#[cfg(feature = "libsql")]
+mod tests {
+    use std::time::Duration;
+
+    use crate::support::test_rig::TestRigBuilder;
+    use crate::support::trace_llm::LlmTrace;
+
+    // -----------------------------------------------------------------------
+    // Test 1: time_parse_and_diff
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn time_parse_and_diff() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/time_parse_diff.json"
+        ))
+        .expect("failed to load time_parse_diff.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Parse a time and compute a diff").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Time tool should have been called twice (parse + diff).
+        let started = rig.tool_calls_started();
+        let time_count = started.iter().filter(|n| n.as_str() == "time").count();
+        assert!(
+            time_count >= 2,
+            "Expected >= 2 time tool calls, got {time_count}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 2: time_parse_invalid
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn time_parse_invalid() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/time_parse_invalid.json"
+        ))
+        .expect("failed to load time_parse_invalid.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Parse an invalid timestamp").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // The time tool call should have failed (invalid timestamp).
+        let completed = rig.tool_calls_completed();
+        let time_results: Vec<_> = completed
+            .iter()
+            .filter(|(name, _)| name == "time")
+            .collect();
+        assert!(!time_results.is_empty(), "Expected time tool to be called");
+        assert!(
+            time_results.iter().any(|(_, ok)| !ok),
+            "Expected at least one failed time call: {time_results:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 3: routine_create_list
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn routine_create_list() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/routine_create_list.json"
+        ))
+        .expect("failed to load routine_create_list.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Create a daily routine and list all routines")
+            .await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Both routine_create and routine_list should have succeeded.
+        let completed = rig.tool_calls_completed();
+        assert!(
+            completed.iter().any(|(n, ok)| n == "routine_create" && *ok),
+            "routine_create should succeed: {completed:?}"
+        );
+        assert!(
+            completed.iter().any(|(n, ok)| n == "routine_list" && *ok),
+            "routine_list should succeed: {completed:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 4: routine_update_delete
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn routine_update_delete() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/routine_update_delete.json"
+        ))
+        .expect("failed to load routine_update_delete.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Create, update, and delete a routine")
+            .await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        let started = rig.tool_calls_started();
+        assert!(
+            started.contains(&"routine_create".to_string()),
+            "routine_create not started"
+        );
+        assert!(
+            started.contains(&"routine_update".to_string()),
+            "routine_update not started"
+        );
+        assert!(
+            started.contains(&"routine_delete".to_string()),
+            "routine_delete not started"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 5: routine_history
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn routine_history() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/routine_history.json"
+        ))
+        .expect("failed to load routine_history.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Create a routine and check its history")
+            .await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        let started = rig.tool_calls_started();
+        assert!(
+            started.contains(&"routine_create".to_string()),
+            "routine_create missing"
+        );
+        assert!(
+            started.contains(&"routine_history".to_string()),
+            "routine_history missing"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 6: job_create_status
+    // -----------------------------------------------------------------------
+    // Uses {{call_cj_1.job_id}} template to forward the dynamic UUID from
+    // create_job's result into job_status's arguments.
+
+    #[tokio::test]
+    async fn job_create_status() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/job_create_status.json"
+        ))
+        .expect("failed to load job_create_status.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Create a job and check its status").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Both tools should have succeeded.
+        let completed = rig.tool_calls_completed();
+        assert!(
+            completed.iter().any(|(n, ok)| n == "create_job" && *ok),
+            "create_job should succeed: {completed:?}"
+        );
+        assert!(
+            completed.iter().any(|(n, ok)| n == "job_status" && *ok),
+            "job_status should succeed: {completed:?}"
+        );
+
+        // Verify tool results contain expected content.
+        let results = rig.tool_results();
+        let create_result = results
+            .iter()
+            .find(|(n, _)| n == "create_job")
+            .expect("create_job result missing");
+        assert!(
+            create_result.1.contains("job_id"),
+            "create_job should return a job_id: {:?}",
+            create_result.1
+        );
+        let status_result = results
+            .iter()
+            .find(|(n, _)| n == "job_status")
+            .expect("job_status result missing");
+        assert!(
+            status_result.1.contains("Test analysis job"),
+            "job_status should return the job title: {:?}",
+            status_result.1
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 7: job_list_cancel
+    // -----------------------------------------------------------------------
+    // Uses {{call_cj_lc.job_id}} template to forward the dynamic UUID from
+    // create_job into cancel_job.
+
+    #[tokio::test]
+    async fn job_list_cancel() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/job_list_cancel.json"
+        ))
+        .expect("failed to load job_list_cancel.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Create a job, list jobs, then cancel it")
+            .await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // All three tools should have succeeded.
+        let completed = rig.tool_calls_completed();
+        assert!(
+            completed.iter().any(|(n, ok)| n == "create_job" && *ok),
+            "create_job should succeed: {completed:?}"
+        );
+        assert!(
+            completed.iter().any(|(n, ok)| n == "list_jobs" && *ok),
+            "list_jobs should succeed: {completed:?}"
+        );
+        assert!(
+            completed.iter().any(|(n, ok)| n == "cancel_job" && *ok),
+            "cancel_job should succeed: {completed:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 8: http_get_with_replay
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn http_get_with_replay() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/tools/http_get_replay.json"
+        ))
+        .expect("failed to load http_get_replay.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Make an http GET request").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // HTTP tool should have succeeded with the replayed exchange.
+        let completed = rig.tool_calls_completed();
+        assert!(
+            completed.iter().any(|(n, ok)| n == "http" && *ok),
+            "http tool should succeed: {completed:?}"
+        );
+
+        rig.shutdown();
+    }
+}