diff --git a/crates/protocols/src/responses.rs b/crates/protocols/src/responses.rs index 52a396b1d5..0da90f9a70 100644 --- a/crates/protocols/src/responses.rs +++ b/crates/protocols/src/responses.rs @@ -573,18 +573,6 @@ impl ResponseUsage { } } -#[derive(Debug, Clone, Default, Deserialize, Serialize, schemars::JsonSchema)] -pub struct ResponsesGetParams { - #[serde(default)] - pub include: Vec, - #[serde(default)] - pub include_obfuscation: Option, - #[serde(default)] - pub starting_after: Option, - #[serde(default)] - pub stream: Option, -} - impl ResponsesUsage { pub fn to_response_usage(&self) -> ResponseUsage { match self { diff --git a/model_gateway/src/routers/grpc/common/responses/handlers.rs b/model_gateway/src/routers/grpc/common/responses/handlers.rs index b94931208e..7f2791c986 100644 --- a/model_gateway/src/routers/grpc/common/responses/handlers.rs +++ b/model_gateway/src/routers/grpc/common/responses/handlers.rs @@ -2,33 +2,12 @@ //! //! These handlers are used by both pipelines for retrieving and cancelling responses. -use axum::response::{IntoResponse, Response}; +use axum::response::Response; use smg_data_connector::ResponseId; use super::ResponsesContext; use crate::routers::error; -/// Implementation for GET /v1/responses/{response_id} -/// -/// Retrieves a stored response from the database. -/// Used by both regular and harmony implementations. -pub(crate) async fn get_response_impl(ctx: &ResponsesContext, response_id: &str) -> Response { - let resp_id = ResponseId::from(response_id); - - // Retrieve response from storage - match ctx.response_storage.get_response(&resp_id).await { - Ok(Some(stored_response)) => axum::Json(stored_response.raw_response).into_response(), - Ok(None) => error::not_found( - "response_not_found", - format!("Response with id '{response_id}' not found"), - ), - Err(e) => error::internal_error( - "retrieve_response_failed", - format!("Failed to retrieve response: {e}"), - ), - } -} - /// Implementation for POST /v1/responses/{response_id}/cancel /// /// Background mode is no longer supported, so this endpoint always returns diff --git a/model_gateway/src/routers/grpc/router.rs b/model_gateway/src/routers/grpc/router.rs index d8dd2813ca..1a4d63350d 100644 --- a/model_gateway/src/routers/grpc/router.rs +++ b/model_gateway/src/routers/grpc/router.rs @@ -6,20 +6,14 @@ use axum::{ response::{IntoResponse, Response}, }; use openai_protocol::{ - chat::ChatCompletionRequest, - classify::ClassifyRequest, - embedding::EmbeddingRequest, - generate::GenerateRequest, - messages::CreateMessageRequest, - responses::{ResponsesGetParams, ResponsesRequest}, + chat::ChatCompletionRequest, classify::ClassifyRequest, embedding::EmbeddingRequest, + generate::GenerateRequest, messages::CreateMessageRequest, responses::ResponsesRequest, }; use tracing::debug; use super::{ common::responses::{ - handlers::{cancel_response_impl, get_response_impl}, - utils::validate_worker_availability, - ResponsesContext, + handlers::cancel_response_impl, utils::validate_worker_availability, ResponsesContext, }, context::SharedComponents, harmony::{serve_harmony_responses, serve_harmony_responses_stream, HarmonyDetector}, @@ -482,15 +476,6 @@ impl RouterTrait for GrpcRouter { self.route_responses_impl(headers, body, model_id).await } - async fn get_response( - &self, - _headers: Option<&HeaderMap>, - response_id: &str, - _params: &ResponsesGetParams, - ) -> Response { - get_response_impl(&self.responses_context, response_id).await - } - async fn cancel_response(&self, _headers: Option<&HeaderMap>, response_id: &str) -> Response { cancel_response_impl(&self.responses_context, response_id).await } diff --git a/model_gateway/src/routers/http/router.rs b/model_gateway/src/routers/http/router.rs index 3cfc87ab02..9052e8a374 100644 --- a/model_gateway/src/routers/http/router.rs +++ b/model_gateway/src/routers/http/router.rs @@ -16,7 +16,7 @@ use openai_protocol::{ embedding::EmbeddingRequest, generate::GenerateRequest, rerank::{RerankRequest, RerankResponse, RerankResult}, - responses::{ResponsesGetParams, ResponsesRequest}, + responses::ResponsesRequest, }; use reqwest::Client; use tokio::sync::mpsc; @@ -450,12 +450,6 @@ impl Router { .unwrap_or_else(|| error::bad_gateway("no_worker_response", "No worker response")) } - // Route a GET request with provided headers to a specific endpoint - async fn route_get_request(&self, headers: Option<&HeaderMap>, endpoint: &str) -> Response { - self.route_simple_request(headers, endpoint, Method::GET) - .await - } - // Route a POST request with empty body to a specific endpoint async fn route_post_empty_request( &self, @@ -732,16 +726,6 @@ impl RouterTrait for Router { .await } - async fn get_response( - &self, - headers: Option<&HeaderMap>, - response_id: &str, - _params: &ResponsesGetParams, - ) -> Response { - let endpoint = format!("v1/responses/{response_id}"); - self.route_get_request(headers, &endpoint).await - } - async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response { let endpoint = format!("v1/responses/{response_id}/cancel"); self.route_post_empty_request(headers, &endpoint).await diff --git a/model_gateway/src/routers/mod.rs b/model_gateway/src/routers/mod.rs index 1cf3818b32..21594aff3b 100644 --- a/model_gateway/src/routers/mod.rs +++ b/model_gateway/src/routers/mod.rs @@ -22,7 +22,7 @@ use openai_protocol::{ RealtimeTranscriptionSessionCreateRequest, }, rerank::RerankRequest, - responses::{ResponsesGetParams, ResponsesRequest}, + responses::ResponsesRequest, }; pub mod anthropic; @@ -38,6 +38,7 @@ pub mod mesh; pub mod openai; pub mod parse; pub mod persistence_utils; +pub mod responses; pub mod router_manager; pub mod tokenize; pub mod worker_selection; @@ -139,16 +140,6 @@ pub trait RouterTrait: Send + Sync + Debug { .into_response() } - /// Retrieve a stored/background response by id - async fn get_response( - &self, - _headers: Option<&HeaderMap>, - _response_id: &str, - _params: &ResponsesGetParams, - ) -> Response { - (StatusCode::NOT_IMPLEMENTED, "Get response not implemented").into_response() - } - /// Cancel a background response by id async fn cancel_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { ( @@ -158,28 +149,6 @@ pub trait RouterTrait: Send + Sync + Debug { .into_response() } - /// Delete a response by id - async fn delete_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { - ( - StatusCode::NOT_IMPLEMENTED, - "Responses delete endpoint not implemented", - ) - .into_response() - } - - /// List input items of a response by id - async fn list_response_input_items( - &self, - _headers: Option<&HeaderMap>, - _response_id: &str, - ) -> Response { - ( - StatusCode::NOT_IMPLEMENTED, - "Responses list input items endpoint not implemented", - ) - .into_response() - } - /// Route embedding requests (OpenAI-compatible /v1/embeddings) async fn route_embeddings( &self, diff --git a/model_gateway/src/routers/openai/responses/mod.rs b/model_gateway/src/routers/openai/responses/mod.rs index e9c6255093..26d808780e 100644 --- a/model_gateway/src/routers/openai/responses/mod.rs +++ b/model_gateway/src/routers/openai/responses/mod.rs @@ -7,7 +7,7 @@ //! - Response accumulation for persistence //! - Tool call detection and output index remapping //! - Input history loading from conversations and response chains -//! - Storage query handlers for response retrieval +//! - Shared helpers for response retrieval-related logic mod accumulator; mod common; @@ -19,85 +19,6 @@ mod utils; // Re-exported for openai::mcp::tool_handler (cross-module dependency) pub(crate) use accumulator::StreamingResponseAccumulator; -// --- Storage query handlers (extracted from router.rs) --- -use axum::{ - http::StatusCode, - response::{IntoResponse, Response}, - Json, -}; pub(crate) use common::{extract_output_index, get_event_type}; pub use non_streaming::handle_non_streaming_response; -use openai_protocol::responses::generate_id; -use serde_json::{json, Value}; -use smg_data_connector::ResponseId; pub use streaming::handle_streaming_response; -use tracing::warn; - -use super::context::ResponsesComponents; -use crate::routers::error; - -/// Fetch a single stored response by ID. -pub(crate) async fn get_response(components: &ResponsesComponents, response_id: &str) -> Response { - let id = ResponseId::from(response_id); - match components.response_storage.get_response(&id).await { - Ok(Some(stored)) => { - let mut response_json = stored.raw_response; - if let Some(obj) = response_json.as_object_mut() { - obj.insert("id".to_string(), json!(id.0)); - } - (StatusCode::OK, Json(response_json)).into_response() - } - Ok(None) => error::not_found( - "not_found", - format!("No response found with id '{response_id}'"), - ), - Err(e) => error::internal_error("storage_error", format!("Failed to get response: {e}")), - } -} - -/// List input items for a stored response. -pub(crate) async fn list_response_input_items( - components: &ResponsesComponents, - response_id: &str, -) -> Response { - let resp_id = ResponseId::from(response_id); - - match components.response_storage.get_response(&resp_id).await { - Ok(Some(stored)) => { - let items = stored.input.as_array().cloned().unwrap_or_default(); - - let items_with_ids: Vec = items - .into_iter() - .map(|mut item| { - if item.get("id").is_none() { - if let Some(obj) = item.as_object_mut() { - obj.insert("id".to_string(), json!(generate_id("msg"))); - } - } - item - }) - .collect(); - - let response_body = json!({ - "object": "list", - "data": items_with_ids, - "first_id": items_with_ids.first().and_then(|v| v.get("id").and_then(|i| i.as_str())), - "last_id": items_with_ids.last().and_then(|v| v.get("id").and_then(|i| i.as_str())), - "has_more": false - }); - - (StatusCode::OK, Json(response_body)).into_response() - } - Ok(None) => error::not_found( - "not_found", - format!("No response found with id '{response_id}'"), - ), - Err(e) => { - warn!("Failed to retrieve input items for {}: {}", response_id, e); - error::internal_error( - "storage_error", - format!("Failed to retrieve input items: {e}"), - ) - } - } -} diff --git a/model_gateway/src/routers/openai/router.rs b/model_gateway/src/routers/openai/router.rs index 9e82b1fb6e..bd33b5b4ce 100644 --- a/model_gateway/src/routers/openai/router.rs +++ b/model_gateway/src/routers/openai/router.rs @@ -10,7 +10,7 @@ use openai_protocol::{ RealtimeClientSecretCreateRequest, RealtimeSessionCreateRequest, RealtimeTranscriptionSessionCreateRequest, }, - responses::{ResponsesGetParams, ResponsesRequest}, + responses::ResponsesRequest, }; use super::{ @@ -173,23 +173,6 @@ impl crate::routers::RouterTrait for OpenAIRouter { responses_route::route_responses(&deps, headers, body, model_id).await } - async fn get_response( - &self, - _headers: Option<&HeaderMap>, - response_id: &str, - _params: &ResponsesGetParams, - ) -> Response { - super::responses::get_response(&self.responses_components, response_id).await - } - - async fn list_response_input_items( - &self, - _headers: Option<&HeaderMap>, - response_id: &str, - ) -> Response { - super::responses::list_response_input_items(&self.responses_components, response_id).await - } - async fn route_realtime_session( &self, headers: Option<&HeaderMap>, diff --git a/model_gateway/src/routers/responses/handlers.rs b/model_gateway/src/routers/responses/handlers.rs new file mode 100644 index 0000000000..ddb2ea25be --- /dev/null +++ b/model_gateway/src/routers/responses/handlers.rs @@ -0,0 +1,106 @@ +use std::sync::Arc; + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use openai_protocol::responses::generate_id; +use serde_json::{json, Value}; +use smg_data_connector::{ResponseId, ResponseStorage}; +use tracing::{info, warn}; + +use crate::routers::error; + +pub async fn get_response( + response_storage: &Arc, + response_id: &str, +) -> Response { + let id = ResponseId::from(response_id); + match response_storage.get_response(&id).await { + Ok(Some(stored)) => (StatusCode::OK, Json(stored.raw_response)).into_response(), + Ok(None) => error::not_found( + "not_found", + format!("No response found with id '{response_id}'"), + ), + Err(e) => error::internal_error("storage_error", format!("Failed to get response: {e}")), + } +} + +pub async fn delete_response( + response_storage: &Arc, + response_id: &str, +) -> Response { + let id = ResponseId::from(response_id); + + match response_storage.get_response(&id).await { + Ok(Some(_)) => match response_storage.delete_response(&id).await { + Ok(()) => { + info!(response_id = %id.0, "Deleted response"); + ( + StatusCode::OK, + Json(json!({ + "id": id.0, + "object": "response.deleted", + "deleted": true, + })), + ) + .into_response() + } + Err(e) => { + error::internal_error("storage_error", format!("Failed to delete response: {e}")) + } + }, + Ok(None) => error::not_found( + "not_found", + format!("No response found with id '{response_id}'"), + ), + Err(e) => error::internal_error("storage_error", format!("Failed to get response: {e}")), + } +} + +pub async fn list_response_input_items( + response_storage: &Arc, + response_id: &str, +) -> Response { + let resp_id = ResponseId::from(response_id); + + match response_storage.get_response(&resp_id).await { + Ok(Some(stored)) => { + let items = stored.input.as_array().cloned().unwrap_or_default(); + + let items_with_ids: Vec = items + .into_iter() + .map(|mut item| { + if item.get("id").is_none() { + if let Some(obj) = item.as_object_mut() { + obj.insert("id".to_string(), json!(generate_id("msg"))); + } + } + item + }) + .collect(); + + let response_body = json!({ + "object": "list", + "data": items_with_ids, + "first_id": items_with_ids.first().and_then(|v| v.get("id").and_then(|i| i.as_str())), + "last_id": items_with_ids.last().and_then(|v| v.get("id").and_then(|i| i.as_str())), + "has_more": false + }); + + (StatusCode::OK, Json(response_body)).into_response() + } + Ok(None) => error::not_found( + "not_found", + format!("No response found with id '{response_id}'"), + ), + Err(e) => { + warn!("Failed to retrieve input items for {}: {}", response_id, e); + error::internal_error( + "storage_error", + format!("Failed to retrieve input items: {e}"), + ) + } + } +} diff --git a/model_gateway/src/routers/responses/mod.rs b/model_gateway/src/routers/responses/mod.rs new file mode 100644 index 0000000000..f0de120c2a --- /dev/null +++ b/model_gateway/src/routers/responses/mod.rs @@ -0,0 +1,8 @@ +//! Shared response management module. +//! +//! This module provides response retrieval operations that are shared +//! across server-level handlers. + +mod handlers; + +pub use handlers::*; diff --git a/model_gateway/src/routers/router_manager.rs b/model_gateway/src/routers/router_manager.rs index 926e06a6eb..aee290b64f 100644 --- a/model_gateway/src/routers/router_manager.rs +++ b/model_gateway/src/routers/router_manager.rs @@ -32,7 +32,7 @@ use openai_protocol::{ RealtimeTranscriptionSessionCreateRequest, }, rerank::RerankRequest, - responses::{ResponsesGetParams, ResponsesRequest}, + responses::ResponsesRequest, }; use serde_json::Value; use tracing::{debug, info, warn}; @@ -623,24 +623,6 @@ impl RouterTrait for RouterManager { } } - async fn get_response( - &self, - headers: Option<&HeaderMap>, - response_id: &str, - params: &ResponsesGetParams, - ) -> Response { - let router = self.select_router_for_request(headers, None); - if let Some(router) = router { - router.get_response(headers, response_id, params).await - } else { - ( - StatusCode::NOT_FOUND, - format!("No router available to get response '{response_id}'"), - ) - .into_response() - } - } - async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response { let router = self.select_router_for_request(headers, None); if let Some(router) = router { @@ -654,33 +636,6 @@ impl RouterTrait for RouterManager { } } - async fn delete_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { - ( - StatusCode::NOT_IMPLEMENTED, - "responses api not yet implemented in inference gateway mode", - ) - .into_response() - } - - async fn list_response_input_items( - &self, - headers: Option<&HeaderMap>, - response_id: &str, - ) -> Response { - // Delegate to the default router (typically http-regular) - // Response storage is shared across all routers via AppContext - let router = self.select_router_for_request(headers, None); - if let Some(router) = router { - router.list_response_input_items(headers, response_id).await - } else { - ( - StatusCode::NOT_FOUND, - "No router available to list response input items", - ) - .into_response() - } - } - async fn route_embeddings( &self, headers: Option<&HeaderMap>, diff --git a/model_gateway/src/server.rs b/model_gateway/src/server.rs index 1c93aa32a0..7d1a1e040f 100644 --- a/model_gateway/src/server.rs +++ b/model_gateway/src/server.rs @@ -28,7 +28,7 @@ use openai_protocol::{ RealtimeTranscriptionSessionCreateRequest, }, rerank::{RerankRequest, V1RerankReqInput}, - responses::{ResponsesGetParams, ResponsesRequest}, + responses::ResponsesRequest, tokenize::{AddTokenizerRequest, DetokenizeRequest, TokenizeRequest}, validated::ValidatedJson, worker::{WorkerSpec, WorkerUpdateRequest}, @@ -65,7 +65,7 @@ use crate::{ get_worker_states, set_global_rate_limit, trigger_graceful_shutdown, update_app_config, }, openai::realtime::ws::RealtimeQueryParams, - parse, + parse, responses as response_handlers, router_manager::RouterManager, tokenize, RouterTrait, }, @@ -290,13 +290,8 @@ async fn v1_classify( async fn v1_responses_get( State(state): State>, Path(response_id): Path, - headers: http::HeaderMap, - Query(params): Query, ) -> Response { - state - .router - .get_response(Some(&headers), &response_id, ¶ms) - .await + response_handlers::get_response(&state.context.response_storage, &response_id).await } async fn v1_responses_cancel( @@ -313,22 +308,15 @@ async fn v1_responses_cancel( async fn v1_responses_delete( State(state): State>, Path(response_id): Path, - headers: http::HeaderMap, ) -> Response { - state - .router - .delete_response(Some(&headers), &response_id) - .await + response_handlers::delete_response(&state.context.response_storage, &response_id).await } async fn v1_responses_list_input_items( State(state): State>, Path(response_id): Path, - headers: http::HeaderMap, ) -> Response { - state - .router - .list_response_input_items(Some(&headers), &response_id) + response_handlers::list_response_input_items(&state.context.response_storage, &response_id) .await } diff --git a/model_gateway/tests/api/api_endpoints_test.rs b/model_gateway/tests/api/api_endpoints_test.rs index 8270ad72df..47dfc82cd0 100644 --- a/model_gateway/tests/api/api_endpoints_test.rs +++ b/model_gateway/tests/api/api_endpoints_test.rs @@ -645,8 +645,6 @@ mod router_policy_tests { #[cfg(test)] mod responses_endpoint_tests { - use reqwest::Client as HttpClient; - use super::*; #[tokio::test] @@ -742,24 +740,24 @@ mod responses_endpoint_tests { let app = ctx.create_app(); - // First create a response to obtain an id let resp_id = "test-get-resp-id-123"; - let payload = json!({ - "input": "Hello Responses API", + use smg_data_connector::{ResponseId, StoredResponse}; + let mut stored_response = StoredResponse::new(None); + stored_response.id = ResponseId::from(resp_id); + stored_response.raw_response = json!({ + "id": resp_id, + "object": "response", + "created_at": 123, "model": "mock-model", - "stream": false, - "store": true, - "background": true, - "request_id": resp_id + "output": [], + "status": "completed" }); - let req = Request::builder() - .method("POST") - .uri("/v1/responses") - .header(CONTENT_TYPE, "application/json") - .body(Body::from(serde_json::to_string(&payload).unwrap())) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); + + ctx.app_context + .response_storage + .store_response(stored_response) + .await + .expect("Failed to store response"); // Retrieve the response let req = Request::builder() @@ -828,7 +826,7 @@ mod responses_endpoint_tests { } #[tokio::test] - async fn test_v1_responses_delete_not_implemented() { + async fn test_v1_responses_delete() { let ctx = AppTestContext::new(vec![MockWorkerConfig { port: 18954, worker_type: WorkerType::Regular, @@ -840,8 +838,24 @@ mod responses_endpoint_tests { let app = ctx.create_app(); - // Test DELETE is not implemented let resp_id = "resp-test-123"; + use smg_data_connector::{ResponseId, StoredResponse}; + let mut stored_response = StoredResponse::new(None); + stored_response.id = ResponseId::from(resp_id); + stored_response.raw_response = json!({ + "id": resp_id, + "object": "response", + "created_at": 123, + "model": "mock-model", + "output": [], + "status": "completed" + }); + + ctx.app_context + .response_storage + .store_response(stored_response) + .await + .expect("Failed to store response"); let req = Request::builder() .method("DELETE") @@ -849,7 +863,22 @@ mod responses_endpoint_tests { .body(Body::empty()) .unwrap(); let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_IMPLEMENTED); + assert_eq!(resp.status(), StatusCode::OK); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let delete_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(delete_json["id"], resp_id); + assert_eq!(delete_json["object"], "response.deleted"); + assert_eq!(delete_json["deleted"], true); + + let req = Request::builder() + .method("GET") + .uri(format!("/v1/responses/{resp_id}")) + .body(Body::empty()) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); ctx.shutdown().await; } @@ -926,7 +955,7 @@ mod responses_endpoint_tests { } #[tokio::test] - async fn test_v1_responses_get_multi_worker_fanout() { + async fn test_v1_responses_get_multi_worker_uses_shared_storage() { // Start two mock workers let ctx = AppTestContext::new(vec![ MockWorkerConfig { @@ -948,26 +977,26 @@ mod responses_endpoint_tests { let app = ctx.create_app(); - // Create a background response with a known id let rid = format!("resp_{}", 18960); // arbitrary unique id - let payload = json!({ - "input": "Hello Responses API", + use smg_data_connector::{ResponseId, StoredResponse}; + let mut stored_response = StoredResponse::new(None); + stored_response.id = ResponseId::from(rid.as_str()); + stored_response.raw_response = json!({ + "id": rid, + "object": "response", + "created_at": 123, "model": "mock-model", - "background": true, - "store": true, - "request_id": rid, + "output": [], + "status": "completed" }); - let req = Request::builder() - .method("POST") - .uri("/v1/responses") - .header(CONTENT_TYPE, "application/json") - .body(Body::from(serde_json::to_string(&payload).unwrap())) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); + ctx.app_context + .response_storage + .store_response(stored_response) + .await + .expect("Failed to store response"); - // Using the router, GET should succeed by fanning out across workers + // Retrieval should succeed regardless of worker count because data layer is authoritative. let req = Request::builder() .method("GET") .uri(format!("/v1/responses/{rid}")) @@ -976,23 +1005,6 @@ mod responses_endpoint_tests { let resp = app.clone().oneshot(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); - // Validate only one worker holds the metadata: direct calls - let client = HttpClient::new(); - let mut ok_count = 0usize; - // Get the actual worker URLs from the context - let worker_urls: Vec = vec![ - "http://127.0.0.1:18960".to_string(), - "http://127.0.0.1:18961".to_string(), - ]; - for url in worker_urls { - let get_url = format!("{url}/v1/responses/{rid}"); - let res = client.get(get_url).send().await.unwrap(); - if res.status() == StatusCode::OK { - ok_count += 1; - } - } - assert_eq!(ok_count, 1, "exactly one worker should store the response"); - ctx.shutdown().await; } diff --git a/model_gateway/tests/routing/test_openai_routing.rs b/model_gateway/tests/routing/test_openai_routing.rs index de457a822c..48f9a6c2f5 100644 --- a/model_gateway/tests/routing/test_openai_routing.rs +++ b/model_gateway/tests/routing/test_openai_routing.rs @@ -21,7 +21,7 @@ use openai_protocol::{ common::StringOrArray, completion::CompletionRequest, generate::GenerateRequest, - responses::{ResponseInput, ResponsesGetParams, ResponsesRequest}, + responses::{ResponseInput, ResponsesRequest}, }; use serde_json::json; use smg::{ @@ -303,25 +303,8 @@ async fn test_openai_router_responses_with_mock() { assert_eq!(output_items2.len(), 1); assert_eq!(output_items2[0]["content"][0]["text"], "mock_output_2"); - let get1 = router - .get_response(None, &stored1.id.0, &ResponsesGetParams::default()) - .await; - assert_eq!(get1.status(), StatusCode::OK); - let get1_body_bytes = axum::body::to_bytes(get1.into_body(), usize::MAX) - .await - .unwrap(); - let get1_json: serde_json::Value = serde_json::from_slice(&get1_body_bytes).unwrap(); - assert_eq!(get1_json, body1); - - let get2 = router - .get_response(None, &stored2.id.0, &ResponsesGetParams::default()) - .await; - assert_eq!(get2.status(), StatusCode::OK); - let get2_body_bytes = axum::body::to_bytes(get2.into_body(), usize::MAX) - .await - .unwrap(); - let get2_json: serde_json::Value = serde_json::from_slice(&get2_body_bytes).unwrap(); - assert_eq!(get2_json, body2); + assert_eq!(stored1.raw_response, body1); + assert_eq!(stored2.raw_response, body2); server.abort(); }