Skip to content
7 changes: 6 additions & 1 deletion crates/core/src/engine/scheduler/rank.rs
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,10 @@ impl RankEngine for SchedulerRank {
} else {
false
};
if suppressed_pending_output && let Some((request_id, _)) = pending_suppression {
if suppressed_pending_output
&& let Some((request_id, _)) = pending_suppression
&& !effects.retired_requests.contains(&request_id)
{
// A final output can be suppressed after the native scheduler has
// already retired its request. Replay still owns its accounting
// until the pass completion is observed, so publish the same
Expand Down Expand Up @@ -991,6 +994,7 @@ mod tests {

assert_eq!(effects.result, CommandResult::Applied);
assert!(effects.suppressed_pending_output);
assert_eq!(effects.retired_requests, vec![request_id]);
assert!(pending.effects.outputs.is_empty());
}

Expand Down Expand Up @@ -1041,6 +1045,7 @@ mod tests {

assert_eq!(effects.result, CommandResult::Noop);
assert!(effects.suppressed_pending_output);
assert_eq!(effects.retired_requests, vec![request_id]);
assert!(pending.effects.outputs.is_empty());
}

Expand Down
19 changes: 16 additions & 3 deletions crates/core/src/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::replay::{
},
};
use anyhow::{Context, Result, anyhow, ensure};
use pyo3::exceptions::PyRuntimeError;
use pyo3::exceptions::{PyMemoryError, PyRuntimeError};
use pyo3::prelude::*;
use pyo3::types::{PyAny, PyDict, PyModule};
use serde::Deserialize;
Expand Down Expand Up @@ -1139,18 +1139,31 @@ fn execute_json(payload: &str, capture_artifacts: bool) -> Result<String> {
serde_json::to_string(&output).context("serializing AISimulate replay output")
}

fn replay_python_error(error: anyhow::Error) -> PyErr {
if error.chain().any(|cause| {
matches!(
cause.downcast_ref::<crate::replay::ReplayError>(),
Some(crate::replay::ReplayError::ResourceLimited(_))
)
}) {
PyMemoryError::new_err(format!("{error:#}"))
} else {
PyRuntimeError::new_err(format!("{error:#}"))
}
}

/// Execute one canonical serialized ReplaySpec and return serialized report JSON.
#[pyfunction]
fn run_replay_json(py: Python<'_>, payload: &str) -> PyResult<String> {
py.allow_threads(|| execute_json(payload, false))
.map_err(|error| PyRuntimeError::new_err(format!("{error:#}")))
.map_err(replay_python_error)
}

/// Execute one fixed aggregated ReplaySpec and return report plus parity artifacts.
#[pyfunction]
fn run_replay_with_artifacts_json(py: Python<'_>, payload: &str) -> PyResult<String> {
py.allow_threads(|| execute_json(payload, true))
.map_err(|error| PyRuntimeError::new_err(format!("{error:#}")))
.map_err(replay_python_error)
}

/// AISimulate native runtime module.
Expand Down
27 changes: 27 additions & 0 deletions crates/core/src/replay/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -456,3 +456,30 @@ The easiest way to think about offline replay is:
4. Record the same request lifecycle timings into `TraceCollector`.

That keeps the harness fast, reproducible, and close to the real scheduler behavior without needing to boot a live runtime.

## Batch report memory

Offline aggregated and disaggregated batch replay discard terminal collector
records after completion callbacks. Disaggregated handoff state is removed only
once both pipelines and the coordinator are quiescent. Active request state and
token timelines still scale with the configured concurrency and output length.

Summary latency distributions retain at most 4096 in-memory samples each, then
spill exact values to anonymous temporary files. Quantiles use the same rounded
rank as detailed reports; eight sequential radix-selection passes require fixed
memory. Counts and quantiles remain exact. Floating-point mean and standard
deviation accumulation can differ at roundoff because completion order replaces
request-ID order. Existing ITL and per-user throughput sketches retain their
existing 0.1% relative quantile error; this change introduces no new sketch.

The four exact sample files together have a 1 GiB limit. Filesystem errors and
that limit stop the replay with a `resource_limited` error, without returning a
partial report. Temporary files close on success, failure, or process exit.
Detailed batch output retains its existing list API and refuses request 100001
before adding its report record. Select summary output or explicitly reduce the
workload when that limit is reached. These are reporting limits, not a total host
memory guarantee: engine state, input traces, trajectory metadata, and other
capture options must also fit the execution budget.

Steppable SDK engines preserve completed-request queries until their reporting
epoch is drained; this batch optimization does not change that API contract.
9 changes: 8 additions & 1 deletion crates/core/src/replay/agg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,11 @@ where
})
}

pub(crate) fn with_sla_thresholds(mut self, sla: crate::replay::SlaThresholds) -> Self {
self.collector.set_sla_thresholds(sla);
self
}

/// Toggle per-request record capture on the underlying collector. When
/// `true`, the final `ReplayReport` returned from `run()` will
/// have `per_request` populated. Default `false` (cheap).
Expand Down Expand Up @@ -341,7 +346,7 @@ where
);
}
self.collector
.on_arrival(uuid, arrival_time_ms, input_length, output_length);
.try_on_arrival(uuid, arrival_time_ms, input_length, output_length)?;
if let Some(context) = request.metadata().replay_context.as_ref() {
self.collector.on_request_context(uuid, context);
}
Expand Down Expand Up @@ -1389,6 +1394,7 @@ where
/// timestamp would exceed that cap; in-flight requests at that point are
/// reported as incomplete.
pub(crate) fn run(mut self) -> anyhow::Result<(TraceCollector, AggRuntimeStats)> {
self.collector.begin_batch_reporting();
self.run_to_completion()?;

self.progress.finish();
Expand All @@ -1406,6 +1412,7 @@ where
self.collector.set_agentic_play_outcomes(outcomes);
}
self.collector.set_runtime_evidence(self.evidence.finish());
self.collector.prepare_batch_report()?;
Ok((self.collector, self.stats))
}
}
Expand Down
39 changes: 31 additions & 8 deletions crates/core/src/replay/disagg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,7 @@ impl DisaggFlowState {
request.metadata_mut().uuid = Some(uuid);
request.metadata_mut().arrival_timestamp_ms = Some(arrival_time_ms);

collector.on_arrival(uuid, arrival_time_ms, input_length, output_length);
collector.try_on_arrival(uuid, arrival_time_ms, input_length, output_length)?;
if let Some(context) = request.metadata().replay_context.as_ref() {
collector.on_request_context(uuid, context);
}
Expand Down Expand Up @@ -1088,6 +1088,11 @@ where
})
}

pub(crate) fn with_sla_thresholds(mut self, sla: crate::replay::SlaThresholds) -> Self {
self.collector.set_sla_thresholds(sla);
self
}

/// Toggle per-request record capture on the underlying collector. When
/// `true`, the final `ReplayReport` returned from `run()` will
/// have `per_request` populated. Default `false` (cheap).
Expand Down Expand Up @@ -1249,7 +1254,23 @@ where
}

fn notify_quiescent(&mut self, uuid: Uuid) -> Result<()> {
self.admission.defer_quiescent(uuid, self.now_ms)
self.admission.defer_quiescent(uuid, self.now_ms)?;
if !self.collector.is_batch_reporting() {
return Ok(());
}
// Both pipelines and the handoff coordinator are now quiescent. Only
// test snapshots retain terminal state; production maps track live work.
let state = self
.flow
.requests
.remove(&uuid)
.ok_or_else(|| anyhow!("missing quiescent request {uuid}"))?;
debug_assert_eq!(state.phase, DisaggPhase::Done);
#[cfg(test)]
self.stats
.request_snapshots
.insert(uuid, state.debug_snapshot());
Ok(())
}

/// Submit a coordinator-owned prefill onto a selected worker.
Expand Down Expand Up @@ -2841,12 +2862,12 @@ where
!matches!(state.phase, DisaggPhase::CleanupPending | DisaggPhase::Done)
);
}
self.stats.request_snapshots = self
.flow
.requests
.iter()
.map(|(uuid, state)| (*uuid, state.debug_snapshot()))
.collect();
self.stats.request_snapshots.extend(
self.flow
.requests
.iter()
.map(|(uuid, state)| (*uuid, state.debug_snapshot())),
);
}
}

Expand Down Expand Up @@ -3334,6 +3355,7 @@ where
/// timestamp would exceed that cap; in-flight requests at that point are
/// reported as incomplete.
pub(crate) fn run(mut self) -> Result<(TraceCollector, DisaggRuntimeStats)> {
self.collector.begin_batch_reporting();
self.run_to_completion()?;

self.progress.finish();
Expand All @@ -3351,6 +3373,7 @@ where
self.collector.set_agentic_play_outcomes(outcomes);
}
self.collector.set_runtime_evidence(self.evidence.finish());
self.collector.prepare_batch_report()?;
Ok((self.collector, self.stats))
}

Expand Down
53 changes: 43 additions & 10 deletions crates/core/src/replay/disagg_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,7 @@ fn run_trace_collect(
let pending = crate::replay::normalize_trace_requests(requests, arrival_speedup_ratio).unwrap();
DisaggRuntime::from_requests(config, router_config, None, pending, ReplayMode::Trace)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand Down Expand Up @@ -706,6 +707,7 @@ fn run_concurrency_collect(
ReplayMode::Concurrency { max_in_flight },
)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand All @@ -720,6 +722,7 @@ fn run_trace_workload_collect(
.unwrap();
DisaggRuntime::new_workload(config, router_config, None, driver, ReplayMode::Trace)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand All @@ -741,6 +744,7 @@ fn run_concurrency_workload_collect(
ReplayMode::Concurrency { max_in_flight },
)
.unwrap()
.with_per_request_records(true)
.run()
.unwrap()
}
Expand Down Expand Up @@ -2318,8 +2322,10 @@ fn source_only_reuse_does_not_reduce_destination_missing_transfer() {
}
}

#[test]
fn test_cancellation_during_transfer_ignores_retired_completion_event() {
#[rstest::rstest]
fn test_cancellation_during_transfer_ignores_retired_completion_event(
#[values(false, true)] batch_reporting: bool,
) {
for mode in [
KvTransferTimingMode::FullPrompt,
KvTransferTimingMode::DestinationMissing,
Expand All @@ -2343,6 +2349,9 @@ fn test_cancellation_during_transfer_ignores_retired_completion_event() {
)
.unwrap()
.with_per_request_records(true);
if batch_reporting {
runtime.collector.begin_batch_reporting();
}

runtime.drain_current_timestamp().unwrap();
for _ in 0..16 {
Expand Down Expand Up @@ -2377,7 +2386,15 @@ fn test_cancellation_during_transfer_ignores_retired_completion_event() {
runtime.advance_now_ms(next);
runtime.drain_current_timestamp().unwrap();
}
assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
if batch_reporting {
assert_eq!(
runtime.stats.request_snapshots[&uuid].phase,
DisaggPhase::Done
);
assert!(!runtime.flow.requests.contains_key(&uuid));
} else {
assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
}
assert_eq!(runtime.total_prefill_count(), 0);
assert_eq!(runtime.total_decode_count(), 0);
assert!(
Expand Down Expand Up @@ -2533,8 +2550,10 @@ fn destination_first_workload_materializes_for_decode_reservation_then_completes
assert_eq!(collector.finish().request_counts.total_output_tokens, 4);
}

#[test]
fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt() {
#[rstest::rstest]
fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt(
#[values(false, true)] batch_reporting: bool,
) {
let mut config = disagg_config();
config.num_prefill_workers = 1;
let trace = Trace {
Expand All @@ -2553,6 +2572,9 @@ fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt() {
let driver = WorkloadDriver::new_trace(trace, 64).unwrap();
let mut runtime =
DisaggRuntime::new_workload(&config, None, None, driver, ReplayMode::Trace).unwrap();
if batch_reporting {
runtime.collector.begin_batch_reporting();
}
runtime.apply_scaling(0, config.num_decode_workers).unwrap();

assert!(runtime.release_ready_arrivals().unwrap());
Expand All @@ -2573,11 +2595,22 @@ fn canceling_worker_waiting_compact_prefill_drops_deferred_prompt() {
.unwrap();
runtime.drain_current_timestamp().unwrap();

assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
assert!(
runtime.state(uuid).unwrap().materialized_tokens().is_err(),
"cancellation should release the deferred request payload"
);
if batch_reporting {
assert_eq!(
runtime.stats.request_snapshots[&uuid].phase,
DisaggPhase::Done
);
assert!(
!runtime.flow.requests.contains_key(&uuid),
"batch cancellation releases request state"
);
} else {
assert_eq!(runtime.state(uuid).unwrap().phase, DisaggPhase::Done);
assert!(
runtime.state(uuid).unwrap().materialized_tokens().is_err(),
"cancellation releases the deferred payload"
);
}
}

#[test]
Expand Down
3 changes: 3 additions & 0 deletions crates/core/src/replay/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ pub enum ReplayError {
#[error("replay reached quiescence with {unfinished_requests} unfinished request(s)")]
Deadlock { unfinished_requests: usize },

#[error("resource_limited: {0}")]
ResourceLimited(String),

#[error("replay invariant violated: {0}")]
Invariant(String),
}
Expand Down
Loading