Skip to content
7 changes: 6 additions & 1 deletion crates/core/src/engine/scheduler/rank.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,10 @@ impl RankEngine for SchedulerRank {
} else {
false
};
if suppressed_pending_output && let Some((request_id, _)) = pending_suppression {
if suppressed_pending_output
&& let Some((request_id, _)) = pending_suppression
&& !effects.retired_requests.contains(&request_id)
{
// A final output can be suppressed after the native scheduler has
// already retired its request. Replay still owns its accounting
// until the pass completion is observed, so publish the same
Expand Down Expand Up @@ -998,6 +1001,7 @@ mod tests {

assert_eq!(effects.result, CommandResult::Applied);
assert!(effects.suppressed_pending_output);
assert_eq!(effects.retired_requests, vec![request_id]);
assert!(pending.effects.outputs.is_empty());
}

Expand Down Expand Up @@ -1048,6 +1052,7 @@ mod tests {

assert_eq!(effects.result, CommandResult::Noop);
assert!(effects.suppressed_pending_output);
assert_eq!(effects.retired_requests, vec![request_id]);
assert!(pending.effects.outputs.is_empty());
}

Expand Down
19 changes: 16 additions & 3 deletions crates/core/src/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use crate::replay::{
},
};
use anyhow::{Context, Result, anyhow, ensure};
use pyo3::exceptions::PyRuntimeError;
use pyo3::exceptions::{PyMemoryError, PyRuntimeError};
use pyo3::prelude::*;
use pyo3::types::{PyAny, PyDict, PyModule};
use serde::Deserialize;
Expand Down Expand Up @@ -1527,18 +1527,31 @@ fn execute_json(payload: &str, capture_artifacts: bool) -> Result<String> {
serde_json::to_string(&output).context("serializing AISimulate replay output")
}

fn replay_python_error(error: anyhow::Error) -> PyErr {
if error.chain().any(|cause| {
matches!(
cause.downcast_ref::<crate::replay::ReplayError>(),
Some(crate::replay::ReplayError::ResourceLimited(_))
)
}) {
PyMemoryError::new_err(format!("{error:#}"))
} else {
PyRuntimeError::new_err(format!("{error:#}"))
}
}

/// Execute one canonical serialized ReplaySpec and return serialized report JSON.
#[pyfunction]
fn run_replay_json(py: Python<'_>, payload: &str) -> PyResult<String> {
py.allow_threads(|| execute_json(payload, false))
.map_err(|error| PyRuntimeError::new_err(format!("{error:#}")))
.map_err(replay_python_error)
}

/// Execute one fixed aggregated ReplaySpec and return report plus parity artifacts.
#[pyfunction]
fn run_replay_with_artifacts_json(py: Python<'_>, payload: &str) -> PyResult<String> {
py.allow_threads(|| execute_json(payload, true))
.map_err(|error| PyRuntimeError::new_err(format!("{error:#}")))
.map_err(replay_python_error)
}

/// AISimulate native runtime module.
Expand Down
31 changes: 31 additions & 0 deletions crates/core/src/replay/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -492,3 +492,34 @@ The easiest way to think about offline replay is:
4. Record the same request lifecycle timings into `TraceCollector`.

That keeps the harness fast, reproducible, and close to the real scheduler behavior without needing to boot a live runtime.

## Batch report memory

Offline aggregated and disaggregated batch replay discard terminal collector
records after completion callbacks. Disaggregated handoff state is removed only
once both pipelines and the coordinator are quiescent. Active request state and
token timelines still scale with the configured concurrency and output length.

Summary latency distributions retain at most 4096 in-memory samples each, then
spill exact values to anonymous temporary files. Quantiles use the same rounded
rank as detailed reports; eight sequential radix-selection passes require fixed
memory. Counts and quantiles remain exact. Floating-point mean and standard
deviation accumulation can differ at roundoff because completion order replaces
request-ID order. Existing ITL and per-user throughput sketches retain their
existing 0.1% relative quantile error; this change introduces no new sketch.

The 4096-sample threshold only changes where values are stored; it never drops
samples or stops the replay. Temporary storage grows with the workload, without
an application-imposed byte or sample cap. Actual filesystem failures stop the
replay with a `resource_limited` error, without returning a partial report.
Only the replay's own anonymous temporary files are closed on success, failure,
or process exit; no existing user files are deleted.

Detailed batch output retains its complete, ordered list API without a request
cap. It continues retaining the records needed to satisfy that API. This change
does not introduce a streaming-output API, reduce the requested workload, or
impose host-resource budgets. Engine state, input traces, detailed records,
trajectory metadata, and other capture options still require host memory.

Steppable SDK engines preserve completed-request queries until their reporting
epoch is drained; this batch optimization does not change that API contract.
9 changes: 8 additions & 1 deletion crates/core/src/replay/agg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,11 @@ where
})
}

pub(crate) fn with_sla_thresholds(mut self, sla: crate::replay::SlaThresholds) -> Self {
self.collector.set_sla_thresholds(sla);
self
}

/// Toggle per-request record capture on the underlying collector. When
/// `true`, the final `ReplayReport` returned from `run()` will
/// have `per_request` populated. Default `false` (cheap).
Expand Down Expand Up @@ -341,7 +346,7 @@ where
);
}
self.collector
.on_arrival(uuid, arrival_time_ms, input_length, output_length);
.try_on_arrival(uuid, arrival_time_ms, input_length, output_length)?;
if let Some(context) = request.metadata().replay_context.as_ref() {
self.collector.on_request_context(uuid, context);
}
Expand Down Expand Up @@ -1393,6 +1398,7 @@ where
/// timestamp would exceed that cap; in-flight requests at that point are
/// reported as incomplete.
pub(crate) fn run(mut self) -> anyhow::Result<(TraceCollector, AggRuntimeStats)> {
self.collector.begin_batch_reporting();
self.run_to_completion()?;

self.progress.finish();
Expand All @@ -1410,6 +1416,7 @@ where
self.collector.set_agentic_play_outcomes(outcomes);
}
self.collector.set_runtime_evidence(self.evidence.finish());
self.collector.prepare_batch_report()?;
Ok((self.collector, self.stats))
}
}
Expand Down
39 changes: 31 additions & 8 deletions crates/core/src/replay/disagg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,7 @@ impl DisaggFlowState {
request.metadata_mut().uuid = Some(uuid);
request.metadata_mut().arrival_timestamp_ms = Some(arrival_time_ms);

collector.on_arrival(uuid, arrival_time_ms, input_length, output_length);
collector.try_on_arrival(uuid, arrival_time_ms, input_length, output_length)?;
if let Some(context) = request.metadata().replay_context.as_ref() {
collector.on_request_context(uuid, context);
}
Expand Down Expand Up @@ -1088,6 +1088,11 @@ where
})
}

pub(crate) fn with_sla_thresholds(mut self, sla: crate::replay::SlaThresholds) -> Self {
self.collector.set_sla_thresholds(sla);
self
}

/// Toggle per-request record capture on the underlying collector. When
/// `true`, the final `ReplayReport` returned from `run()` will
/// have `per_request` populated. Default `false` (cheap).
Expand Down Expand Up @@ -1249,7 +1254,23 @@ where
}

fn notify_quiescent(&mut self, uuid: Uuid) -> Result<()> {
self.admission.defer_quiescent(uuid, self.now_ms)
self.admission.defer_quiescent(uuid, self.now_ms)?;
if !self.collector.is_batch_reporting() {
return Ok(());
}
// Both pipelines and the handoff coordinator are now quiescent. Only
// test snapshots retain terminal state; production maps track live work.
let state = self
.flow
.requests
.remove(&uuid)
.ok_or_else(|| anyhow!("missing quiescent request {uuid}"))?;
debug_assert_eq!(state.phase, DisaggPhase::Done);
#[cfg(test)]
self.stats
.request_snapshots
.insert(uuid, state.debug_snapshot());
Ok(())
}

/// Submit a coordinator-owned prefill onto a selected worker.
Expand Down Expand Up @@ -2845,12 +2866,12 @@ where
!matches!(state.phase, DisaggPhase::CleanupPending | DisaggPhase::Done)
);
}
self.stats.request_snapshots = self
.flow
.requests
.iter()
.map(|(uuid, state)| (*uuid, state.debug_snapshot()))
.collect();
self.stats.request_snapshots.extend(
self.flow
.requests
.iter()
.map(|(uuid, state)| (*uuid, state.debug_snapshot())),
);
}
}

Expand Down Expand Up @@ -3338,6 +3359,7 @@ where
/// timestamp would exceed that cap; in-flight requests at that point are
/// reported as incomplete.
pub(crate) fn run(mut self) -> Result<(TraceCollector, DisaggRuntimeStats)> {
self.collector.begin_batch_reporting();
self.run_to_completion()?;

self.progress.finish();
Expand All @@ -3355,6 +3377,7 @@ where
self.collector.set_agentic_play_outcomes(outcomes);
}
self.collector.set_runtime_evidence(self.evidence.finish());
self.collector.prepare_batch_report()?;
Ok((self.collector, self.stats))
}

Expand Down
53 changes: 43 additions & 10 deletions crates/core/src/replay/disagg_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,7 @@ fn run_trace_collect(
let pending = crate::replay::normalize_trace_requests(requests, arrival_speedup_ratio).unwrap();
DisaggRuntime::from_requests(config, router_config, None, pending, ReplayMode::Trace)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand Down Expand Up @@ -707,6 +708,7 @@ fn run_concurrency_collect(
ReplayMode::Concurrency { max_in_flight },
)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand All @@ -721,6 +723,7 @@ fn run_trace_workload_collect(
.unwrap();
DisaggRuntime::new_workload(config, router_config, None, driver, ReplayMode::Trace)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand All @@ -742,6 +745,7 @@ fn run_concurrency_workload_collect(
ReplayMode::Concurrency { max_in_flight },
)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand Down Expand Up @@ -2319,8 +2323,10 @@ fn source_only_reuse_does_not_reduce_destination_missing_transfer() {
}
}

#[test]
fn test_cancellation_during_transfer_ignores_retired_completion_event() {
#[rstest::rstest]
fn test_cancellation_during_transfer_ignores_retired_completion_event(
#[values(false, true)] batch_reporting: bool,
) {
for mode in [
KvTransferTimingMode::FullPrompt,
KvTransferTimingMode::DestinationMissing,
Expand All @@ -2344,6 +2350,9 @@ fn test_cancellation_during_transfer_ignores_retired_completion_event() {
)
.unwrap()
.with_per_request_records(true);
if batch_reporting {
runtime.collector.begin_batch_reporting();
}

runtime.drain_current_timestamp().unwrap();
for _ in 0..16 {
Expand Down Expand Up @@ -2378,7 +2387,15 @@ fn test_cancellation_during_transfer_ignores_retired_completion_event() {
runtime.advance_now_ms(next);
runtime.drain_current_timestamp().unwrap();
}
assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
if batch_reporting {
assert_eq!(
runtime.stats.request_snapshots[&uuid].phase,
DisaggPhase::Done
);
assert!(!runtime.flow.requests.contains_key(&uuid));
} else {
assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
}
assert_eq!(runtime.total_prefill_count(), 0);
assert_eq!(runtime.total_decode_count(), 0);
assert!(
Expand Down Expand Up @@ -2534,8 +2551,10 @@ fn destination_first_workload_materializes_for_decode_reservation_then_completes
assert_eq!(collector.finish().request_counts.total_output_tokens, 4);
}

#[test]
fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt() {
#[rstest::rstest]
fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt(
#[values(false, true)] batch_reporting: bool,
) {
let mut config = disagg_config();
config.num_prefill_workers = 1;
let trace = Trace {
Expand All @@ -2554,6 +2573,9 @@ fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt() {
let driver = WorkloadDriver::new_trace(trace, 64).unwrap();
let mut runtime =
DisaggRuntime::new_workload(&config, None, None, driver, ReplayMode::Trace).unwrap();
if batch_reporting {
runtime.collector.begin_batch_reporting();
}
runtime.apply_scaling(0, config.num_decode_workers).unwrap();

assert!(runtime.release_ready_arrivals().unwrap());
Expand All @@ -2574,11 +2596,22 @@ fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt() {
.unwrap();
runtime.drain_current_timestamp().unwrap();

assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
assert!(
runtime.state(uuid).unwrap().materialized_tokens().is_err(),
"cancellation should release the deferred request payload"
);
if batch_reporting {
assert_eq!(
runtime.stats.request_snapshots[&uuid].phase,
DisaggPhase::Done
);
assert!(
!runtime.flow.requests.contains_key(&uuid),
"batch cancellation releases request state"
);
} else {
assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
assert!(
runtime.state(uuid).unwrap().materialized_tokens().is_err(),
"cancellation releases the deferred payload"
);
}
}

#[test]
Expand Down
3 changes: 3 additions & 0 deletions crates/core/src/replay/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ pub enum ReplayError {
#[error("replay reached quiescence with {unfinished_requests} unfinished request(s)")]
Deadlock { unfinished_requests: usize },

#[error("resource_limited: {0}")]
ResourceLimited(String),

#[error("replay invariant violated: {0}")]
Invariant(String),
}
Expand Down
Loading
Loading