Repository navigation
feat(decisions): add bounded admission and local cancellation primitives #15852
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
84532a7
feat(decisions): add bounded admission and local cancellation primitives
biswapanda 43ff1c5
docs(decisions): define cooperative dispatch shutdown
biswapanda 3cf380a
fix(decisions): bound deadline cancellation grace
biswapanda 6352cbc
fix(decisions): return deadlines promptly and observe overdue cleanup
biswapanda 923602d
test(decisions): simplify deadline task spawning
biswapanda 17ffd05
fix(decisions): retain request span during cleanup
biswapanda c37b074
fix(decisions): keep cleanup diagnostics in request logs
biswapanda File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,352 @@ | ||
| // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| use std::sync::Arc; | ||
|
|
||
| use dynamo_runtime::engine::AsyncEngineContext; | ||
| use tokio::sync::{OwnedSemaphorePermit, Semaphore, TryAcquireError}; | ||
| use tracing::Instrument; | ||
|
|
||
| #[derive(Debug, PartialEq, Eq)] | ||
| pub(crate) enum AdmissionError { | ||
| InvalidBranchCount, | ||
| RequestTooLarge, | ||
| CapacityExhausted, | ||
| AdmissionClosed, | ||
| } | ||
|
|
||
| #[cfg_attr( | ||
| not(test), | ||
| expect(dead_code, reason = "Consumed by decision execution integration") | ||
| )] | ||
| /// Reject excess load immediately, matching frontend admission rather than queueing. | ||
| /// The caller must initialize the semaphore and `limit` from the same bounded configuration. | ||
| pub(crate) fn acquire_admission( | ||
| admission: Arc<Semaphore>, | ||
| branches: u32, | ||
| limit: usize, | ||
| ) -> Result<OwnedSemaphorePermit, AdmissionError> { | ||
| if branches == 0 { | ||
| return Err(AdmissionError::InvalidBranchCount); | ||
| } | ||
| if branches as usize > limit { | ||
| return Err(AdmissionError::RequestTooLarge); | ||
| } | ||
| admission | ||
| .try_acquire_many_owned(branches) | ||
| .map_err(|cause| match cause { | ||
| TryAcquireError::NoPermits => AdmissionError::CapacityExhausted, | ||
| TryAcquireError::Closed => AdmissionError::AdmissionClosed, | ||
| }) | ||
| } | ||
|
|
||
| #[cfg_attr( | ||
| not(test), | ||
| expect(dead_code, reason = "Consumed by decision execution integration") | ||
| )] | ||
| pub(crate) async fn run_until_killed<T>( | ||
| context: &dyn AsyncEngineContext, | ||
| operation: impl std::future::Future<Output = T>, | ||
| ) -> Option<T> { | ||
| tokio::pin!(operation); | ||
| tokio::select! { | ||
| // Preserve completed results when completion and cancellation are both ready. | ||
| biased; | ||
| result = &mut operation => Some(result), | ||
| _ = context.killed() => None, | ||
|
biswapanda marked this conversation as resolved.
Resolved
|
||
| } | ||
| } | ||
|
|
||
| #[cfg_attr( | ||
| not(test), | ||
| expect(dead_code, reason = "Consumed by decision execution integration") | ||
| )] | ||
| pub(crate) async fn spawn_blocking_with_permit<T, E, F>( | ||
| permit: OwnedSemaphorePermit, | ||
| task: F, | ||
| ) -> Result<Result<(T, OwnedSemaphorePermit), E>, tokio::task::JoinError> | ||
| where | ||
| T: Send + 'static, | ||
| E: Send + 'static, | ||
| F: FnOnce() -> Result<T, E> + Send + 'static, | ||
| { | ||
| tokio::task::spawn_blocking(move || task().map(|output| (output, permit))).await | ||
| } | ||
|
|
||
| #[derive(Debug)] | ||
| pub(crate) enum DispatchWaitError { | ||
| #[cfg_attr( | ||
| not(test), | ||
| expect(dead_code, reason = "Inspected by the decision response adapter") | ||
| )] | ||
| Task(tokio::task::JoinError), | ||
| Deadline, | ||
| } | ||
|
|
||
| #[cfg_attr( | ||
| not(test), | ||
| expect(dead_code, reason = "Consumed by decision execution integration") | ||
| )] | ||
| /// The task must own admission until its work completes and should observe `parent.kill()`. | ||
| /// Cleanup runs separately so the response does not wait for the cancellation grace. | ||
| /// Local completion does not establish physical GPU settlement. | ||
| pub(crate) async fn dispatch_with_deadline<T: Send + 'static>( | ||
| mut task: tokio::task::JoinHandle<T>, | ||
| deadline: tokio::time::Instant, | ||
| parent: Arc<dyn AsyncEngineContext>, | ||
| ) -> Result<T, DispatchWaitError> { | ||
| match tokio::time::timeout_at(deadline, &mut task).await { | ||
|
biswapanda marked this conversation as resolved.
Resolved
|
||
| Ok(Ok(output)) => Ok(output), | ||
| Ok(Err(cause)) => { | ||
| parent.kill(); | ||
| Err(DispatchWaitError::Task(cause)) | ||
| } | ||
| Err(_) => { | ||
| parent.kill(); | ||
| tokio::spawn(drain_cancelled_task(task).in_current_span()); | ||
| Err(DispatchWaitError::Deadline) | ||
| } | ||
| } | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } | ||
|
|
||
| async fn drain_cancelled_task<T: Send + 'static>(mut task: tokio::task::JoinHandle<T>) { | ||
| let result = match tokio::time::timeout(std::time::Duration::from_secs(1), &mut task).await { | ||
| Ok(result) => result, | ||
| Err(_) => { | ||
| tracing::warn!( | ||
| "Decision task exceeded cancellation grace; retaining admission until completion" | ||
| ); | ||
| task.await | ||
| } | ||
| }; | ||
| if let Err(cause) = result { | ||
| tracing::error!(error = %cause, "Cancelled decision task failed during cleanup"); | ||
| } | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use super::*; | ||
| use dynamo_runtime::{engine::AsyncEngineContextProvider, pipeline::Context}; | ||
| use std::{future::pending, time::Duration}; | ||
| use tokio::sync::oneshot; | ||
|
|
||
| #[test] | ||
| fn admission_distinguishes_unfulfillable_requests_from_busy_capacity() { | ||
| let admission = Arc::new(Semaphore::new(2)); | ||
| assert_eq!( | ||
| acquire_admission(admission.clone(), 3, 2).unwrap_err(), | ||
| AdmissionError::RequestTooLarge, | ||
| ); | ||
| let permit = acquire_admission(admission.clone(), 2, 2).unwrap(); | ||
| assert_eq!( | ||
| acquire_admission(admission.clone(), 1, 2).unwrap_err(), | ||
| AdmissionError::CapacityExhausted, | ||
| ); | ||
| drop(permit); | ||
| assert!(acquire_admission(admission, 1, 2).is_ok()); | ||
| } | ||
|
|
||
| #[test] | ||
| fn admission_rejects_empty_and_closed_requests_without_reporting_overload() { | ||
| let admission = Arc::new(Semaphore::new(1)); | ||
| assert_eq!( | ||
| acquire_admission(admission.clone(), 0, 1).unwrap_err(), | ||
| AdmissionError::InvalidBranchCount, | ||
| ); | ||
| admission.close(); | ||
| assert_eq!( | ||
| acquire_admission(admission, 1, 1).unwrap_err(), | ||
| AdmissionError::AdmissionClosed, | ||
| ); | ||
| } | ||
|
|
||
| #[tokio::test(start_paused = true)] | ||
| async fn deadline_returns_without_waiting_for_noncooperative_work() { | ||
| let parent = Context::new(()).context(); | ||
| let (release_tx, release_rx) = oneshot::channel(); | ||
| let task = tokio::spawn(release_rx); | ||
| let start = tokio::time::Instant::now(); | ||
| let deadline = start + Duration::from_secs(2); | ||
| let result = dispatch_with_deadline(task, deadline, parent.clone()).await; | ||
| assert!(matches!(result, Err(DispatchWaitError::Deadline))); | ||
| assert!(parent.is_killed()); | ||
| assert_eq!(tokio::time::Instant::now(), deadline); | ||
| release_tx.send(()).unwrap(); | ||
| } | ||
|
|
||
| #[tokio::test(start_paused = true)] | ||
| async fn graceful_stop_preserves_inflight_results() { | ||
| let parent = Context::new(()).context(); | ||
| let task_context = parent.clone(); | ||
| let (result_tx, result_rx) = oneshot::channel(); | ||
| let task = | ||
| tokio::spawn(async move { run_until_killed(task_context.as_ref(), result_rx).await }); | ||
| parent.stop_generating(); | ||
| tokio::task::yield_now().await; | ||
| assert!(!task.is_finished()); | ||
| result_tx.send(42).unwrap(); | ||
| assert_eq!(task.await.unwrap().unwrap().unwrap(), 42); | ||
| } | ||
|
|
||
| #[tokio::test(start_paused = true)] | ||
| async fn expired_deadline_kills_and_drains_local_work() { | ||
| let parent = Context::new(()).context(); | ||
| let worker_context = parent.clone(); | ||
| let admission = Arc::new(Semaphore::new(1)); | ||
| let permit = admission.clone().acquire_owned().await.unwrap(); | ||
| let task = tokio::spawn(async move { | ||
| let _permit = permit; | ||
| run_until_killed(worker_context.as_ref(), pending::<()>()).await | ||
| }); | ||
| let result = | ||
| dispatch_with_deadline(task, tokio::time::Instant::now(), parent.clone()).await; | ||
| assert!(matches!(result, Err(DispatchWaitError::Deadline))); | ||
| assert!(parent.is_killed()); | ||
| let permit = tokio::time::timeout(Duration::from_secs(1), admission.acquire_owned()) | ||
| .await | ||
| .unwrap() | ||
| .unwrap(); | ||
| drop(permit); | ||
| } | ||
|
|
||
| #[tokio::test(start_paused = true)] | ||
| async fn overdue_cleanup_is_logged_without_a_metric_and_keeps_admission() { | ||
| let _interest_guard = crate::test_utils::tracing_interest_guard(); | ||
| let log = tempfile::NamedTempFile::new().unwrap(); | ||
| let subscriber = tracing_subscriber::fmt() | ||
| .with_ansi(false) | ||
| .without_time() | ||
| .with_writer(Arc::new(log.reopen().unwrap())) | ||
| .finish(); | ||
| let _subscriber = tracing::subscriber::set_default(subscriber); | ||
| let registry = prometheus::Registry::new(); | ||
| super::super::metrics::Metrics::new() | ||
| .register(®istry) | ||
| .unwrap(); | ||
| assert!( | ||
| !registry | ||
| .gather() | ||
| .iter() | ||
| .any(|family| { family.name().ends_with("decision_cleanup_overruns_total") }) | ||
| ); | ||
| let parent = Context::new(()).context(); | ||
| let admission = Arc::new(Semaphore::new(1)); | ||
| let permit = admission.clone().acquire_owned().await.unwrap(); | ||
| let (release_tx, release_rx) = oneshot::channel(); | ||
| let task = tokio::spawn(async move { | ||
| let _permit = permit; | ||
| let _ = release_rx.await; | ||
| }); | ||
| let result = dispatch_with_deadline(task, tokio::time::Instant::now(), parent) | ||
| .instrument(tracing::info_span!( | ||
| "http-request", | ||
| request_id = "cleanup-request" | ||
| )) | ||
| .await; | ||
| assert!(matches!(result, Err(DispatchWaitError::Deadline))); | ||
| tokio::task::yield_now().await; | ||
| tokio::time::advance(Duration::from_secs(1)).await; | ||
| tokio::task::yield_now().await; | ||
| let output = std::fs::read_to_string(log.path()).unwrap(); | ||
| assert!(output.contains("exceeded cancellation grace"), "{output}"); | ||
| assert!( | ||
| output.contains("request_id=\"cleanup-request\""), | ||
| "{output}" | ||
| ); | ||
| assert_eq!(admission.available_permits(), 0); | ||
| release_tx.send(()).unwrap(); | ||
| let permit = tokio::time::timeout(Duration::from_secs(1), admission.acquire_owned()) | ||
| .await | ||
| .unwrap() | ||
| .unwrap(); | ||
| drop(permit); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn finished_dispatch_preserves_output_without_killing_context() { | ||
| let parent = Context::new(()).context(); | ||
| let result = dispatch_with_deadline( | ||
| tokio::spawn(async { 42 }), | ||
| tokio::time::Instant::now() + Duration::from_secs(1), | ||
| parent.clone(), | ||
| ) | ||
| .await | ||
| .unwrap(); | ||
| assert_eq!(result, 42); | ||
| assert!(!parent.is_killed()); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn failed_dispatch_kills_context() { | ||
| let parent = Context::new(()).context(); | ||
| let task = tokio::spawn(pending::<()>()); | ||
| task.abort(); | ||
| let result = dispatch_with_deadline( | ||
| task, | ||
| tokio::time::Instant::now() + Duration::from_secs(1), | ||
| parent.clone(), | ||
| ) | ||
| .await; | ||
| match result { | ||
| Err(DispatchWaitError::Task(cause)) => assert!(cause.is_cancelled()), | ||
| _ => panic!("cancelled task should report its join error"), | ||
| } | ||
| assert!(parent.is_killed()); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn parent_kill_interrupts_pending_branch_work() { | ||
| let parent = Context::new(()); | ||
| let context = parent.context(); | ||
| let task_context = context.clone(); | ||
| let task = | ||
| tokio::spawn( | ||
| async move { run_until_killed(task_context.as_ref(), pending::<()>()).await }, | ||
| ); | ||
| context.kill(); | ||
| assert!(task.await.unwrap().is_none()); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn blocking_preflight_owns_admission_until_it_finishes() { | ||
| let semaphore = Arc::new(Semaphore::new(1)); | ||
| let permit = semaphore.clone().acquire_owned().await.unwrap(); | ||
| let (started_tx, started_rx) = oneshot::channel(); | ||
| let (release_tx, release_rx) = oneshot::channel(); | ||
| let task = tokio::spawn(spawn_blocking_with_permit(permit, move || { | ||
| started_tx.send(()).unwrap(); | ||
| release_rx.blocking_recv().unwrap(); | ||
| Ok::<_, ()>(()) | ||
| })); | ||
| started_rx.await.unwrap(); | ||
| assert!(semaphore.clone().try_acquire_owned().is_err()); | ||
| release_tx.send(()).unwrap(); | ||
| let (_, returned_permit) = task.await.unwrap().unwrap().unwrap(); | ||
| drop(returned_permit); | ||
| assert!(semaphore.try_acquire_owned().is_ok()); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn abandoned_preflight_keeps_admission_until_blocking_work_finishes() { | ||
| let semaphore = Arc::new(Semaphore::new(1)); | ||
| let permit = semaphore.clone().acquire_owned().await.unwrap(); | ||
| let (started_tx, started_rx) = oneshot::channel(); | ||
| let (release_tx, release_rx) = oneshot::channel(); | ||
| let task = tokio::spawn(spawn_blocking_with_permit(permit, move || { | ||
| started_tx.send(()).unwrap(); | ||
| release_rx.blocking_recv().unwrap(); | ||
| Ok::<_, ()>(()) | ||
| })); | ||
| started_rx.await.unwrap(); | ||
| task.abort(); | ||
| assert!(task.await.unwrap_err().is_cancelled()); | ||
| assert!(semaphore.clone().try_acquire_owned().is_err()); | ||
| release_tx.send(()).unwrap(); | ||
| let permit = tokio::time::timeout(Duration::from_secs(2), semaphore.acquire_owned()) | ||
| .await | ||
| .unwrap() | ||
| .unwrap(); | ||
| drop(permit); | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.