diff --git a/crates/protocols/src/completion.rs b/crates/protocols/src/completion.rs index e99c41fb88..f1fbe007a0 100644 --- a/crates/protocols/src/completion.rs +++ b/crates/protocols/src/completion.rs @@ -2,37 +2,67 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use validator::Validate; -use super::common::*; +use super::{ + common::*, + sampling_params::{validate_top_k_value, validate_top_p_value}, +}; +use crate::validated::Normalizable; // ============================================================================ // Completions API (v1/completions) - DEPRECATED but still supported // ============================================================================ #[serde_with::skip_serializing_none] -#[derive(Debug, Clone, Deserialize, Serialize, schemars::JsonSchema)] +#[derive(Debug, Clone, Deserialize, Serialize, Validate, schemars::JsonSchema)] +#[validate(schema(function = "validate_completion_cross_parameters"))] pub struct CompletionRequest { /// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang) pub model: String, /// The prompt(s) to generate completions for + #[validate(custom(function = "validate_completion_prompt"))] pub prompt: StringOrArray, - /// The suffix that comes after a completion of inserted text - pub suffix: Option, + /// Generates `best_of` completions server-side and returns the "best" + #[validate(range(min = 0, max = 20))] + pub best_of: Option, - /// The maximum number of tokens to generate - pub max_tokens: Option, + /// Echo back the prompt in addition to the completion + #[serde(default)] + pub echo: bool, - /// What sampling temperature to use, between 0 and 2 - pub temperature: Option, + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, - /// An alternative to sampling with temperature (nucleus sampling) - pub top_p: Option, + /// Modify the likelihood of specified tokens appearing in the completion + pub logit_bias: Option>, + + /// Include the log probabilities on the `logprobs` most likely tokens + #[validate(range(min = 0, max = 5))] + pub logprobs: Option, + + /// The maximum number of tokens to generate + #[validate(range(min = 0))] + pub max_tokens: Option, /// How many completions to generate for each prompt + #[validate(range(min = 1, max = 128))] pub n: Option, + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + + /// If specified, our system will make a best effort to sample deterministically + pub seed: Option, + + /// Up to 4 sequences where the API will stop generating further tokens + #[validate(custom(function = "validate_stop"))] + pub stop: Option, + /// Whether to stream back partial progress #[serde(default)] pub stream: bool, @@ -40,45 +70,37 @@ pub struct CompletionRequest { /// Options for streaming response pub stream_options: Option, - /// Include the log probabilities on the logprobs most likely tokens - pub logprobs: Option, - - /// Echo back the prompt in addition to the completion - #[serde(default)] - pub echo: bool, - - /// Up to 4 sequences where the API will stop generating further tokens - pub stop: Option, - - /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far - pub presence_penalty: Option, - - /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far - pub frequency_penalty: Option, + /// The suffix that comes after a completion of inserted text + pub suffix: Option, - /// Generates best_of completions server-side and returns the "best" - pub best_of: Option, + /// What sampling temperature to use, between 0 and 2 + #[validate(range(min = 0.0, max = 2.0))] + pub temperature: Option, - /// Modify the likelihood of specified tokens appearing in the completion - pub logit_bias: Option>, + /// An alternative to sampling with temperature (nucleus sampling) + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, /// A unique identifier representing your end-user pub user: Option, - /// If specified, our system will make a best effort to sample deterministically - pub seed: Option, - - // -------- Engine Specific Sampling Parameters -------- + // ============================================================================= + // Engine-Specific Sampling Parameters + // ============================================================================= /// Top-k sampling parameter (-1 to disable) + #[validate(custom(function = "validate_top_k_value"))] pub top_k: Option, /// Min-p nucleus sampling parameter + #[validate(range(min = 0.0, max = 1.0))] pub min_p: Option, /// Minimum number of tokens to generate + #[validate(range(min = 1))] pub min_tokens: Option, /// Repetition penalty for reducing repetitive text + #[validate(range(min = 0.0, max = 2.0))] pub repetition_penalty: Option, /// Regex constraint for output generation @@ -123,6 +145,69 @@ pub struct CompletionRequest { pub other: Map, } +impl Normalizable for CompletionRequest {} + +fn validate_completion_prompt(prompt: &StringOrArray) -> Result<(), validator::ValidationError> { + match prompt { + StringOrArray::String(_) => {} + StringOrArray::Array(arr) => { + if arr.is_empty() { + let mut error = validator::ValidationError::new("prompt_empty"); + error.message = Some("prompt array cannot be empty".into()); + return Err(error); + } + } + } + + Ok(()) +} + +fn validate_completion_cross_parameters( + req: &CompletionRequest, +) -> Result<(), validator::ValidationError> { + if req.stream_options.is_some() && !req.stream { + let mut error = validator::ValidationError::new("stream_options_requires_stream"); + error.message = + Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into()); + return Err(error); + } + + if let (Some(min), Some(max)) = (req.min_tokens, req.max_tokens) { + if min > max { + let mut error = validator::ValidationError::new("min_tokens_exceeds_max"); + error.message = Some("min_tokens cannot exceed max_tokens".into()); + return Err(error); + } + } + + let constraint_count = + req.regex.is_some() as u8 + req.ebnf.is_some() as u8 + req.json_schema.is_some() as u8; + if constraint_count > 1 { + let mut error = validator::ValidationError::new("multiple_constraints"); + error.message = Some( + "only one structured output constraint (regex, ebnf, or json_schema) can be active at a time" + .into(), + ); + return Err(error); + } + + if let (Some(best_of), Some(n)) = (req.best_of, req.n) { + if best_of <= n { + let mut error = validator::ValidationError::new("best_of_less_than_n"); + error.message = Some("best_of must be greater than n".into()); + return Err(error); + } + } + + if req.stream && req.best_of.is_some() { + let mut error = validator::ValidationError::new("best_of_not_supported_with_stream"); + error.message = Some("best_of is not supported when stream is enabled".into()); + return Err(error); + } + + Ok(()) +} + impl GenerationRequest for CompletionRequest { fn is_stream(&self) -> bool { self.stream diff --git a/model_gateway/src/routers/grpc/common/stages/dispatch_metadata.rs b/model_gateway/src/routers/grpc/common/stages/dispatch_metadata.rs index 2c62226a6c..4c6494d24d 100644 --- a/model_gateway/src/routers/grpc/common/stages/dispatch_metadata.rs +++ b/model_gateway/src/routers/grpc/common/stages/dispatch_metadata.rs @@ -29,6 +29,7 @@ impl PipelineStage for DispatchMetadataStage { let request_id = proto_request.request_id().to_string(); let model = match &ctx.input.request_type { RequestType::Chat(req) => req.model.clone(), + RequestType::Completion(req) => req.model.clone(), RequestType::Generate(_req) => { // Generate requests don't have a model field // Use model_id from input diff --git a/model_gateway/src/routers/grpc/context.rs b/model_gateway/src/routers/grpc/context.rs index d770228232..3b9dea48ae 100644 --- a/model_gateway/src/routers/grpc/context.rs +++ b/model_gateway/src/routers/grpc/context.rs @@ -11,6 +11,7 @@ use llm_tokenizer::{stop::StopSequenceDecoder, traits::Tokenizer, TokenizerRegis use openai_protocol::{ chat::{ChatCompletionRequest, ChatCompletionResponse}, classify::{ClassifyRequest, ClassifyResponse}, + completion::{CompletionRequest, CompletionResponse}, embedding::{EmbeddingRequest, EmbeddingResponse}, generate::{GenerateRequest, GenerateResponse}, messages::{CreateMessageRequest, Message}, @@ -50,6 +51,7 @@ pub(crate) struct RequestInput { pub(crate) enum RequestType { Chat(Arc), Generate(Arc), + Completion(Arc), Responses(Arc), Embedding(Arc), Classify(Arc), @@ -61,6 +63,7 @@ impl std::fmt::Display for RequestType { match self { Self::Chat(_) => write!(f, "Chat"), Self::Generate(_) => write!(f, "Generate"), + Self::Completion(_) => write!(f, "Completion"), Self::Responses(_) => write!(f, "Responses"), Self::Embedding(_) => write!(f, "Embedding"), Self::Classify(_) => write!(f, "Classify"), @@ -69,6 +72,19 @@ impl std::fmt::Display for RequestType { } } +impl std::fmt::Display for FinalResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Chat(_) => write!(f, "Chat"), + Self::Generate(_) => write!(f, "Generate"), + Self::Completion(_) => write!(f, "Completion"), + Self::Embedding(_) => write!(f, "Embedding"), + Self::Classify(_) => write!(f, "Classify"), + Self::Messages(_) => write!(f, "Messages"), + } + } +} + /// Shared components (injected once at creation) pub(crate) struct SharedComponents { pub tokenizer_registry: Arc, @@ -254,6 +270,24 @@ impl RequestContext { } } + /// Create context for completion request + pub fn for_completion( + request: Arc, + headers: Option, + model_id: String, + components: Arc, + ) -> Self { + Self { + input: RequestInput { + request_type: RequestType::Completion(request), + headers, + model_id, + }, + components, + state: ProcessingState::default(), + } + } + /// Create context for Responses API request pub fn for_responses( request: Arc, @@ -374,6 +408,38 @@ impl RequestContext { } } + /// Get completion request (panics if not completion) + #[expect( + dead_code, + reason = "ref accessor provided for API completeness alongside Arc accessor" + )] + #[expect( + clippy::panic, + reason = "typed accessor: caller guarantees variant via RequestType construction" + )] + pub fn completion_request(&self) -> &CompletionRequest { + match &self.input.request_type { + RequestType::Completion(req) => req.as_ref(), + _ => panic!("Expected completion request"), + } + } + + /// Get Arc clone of completion request (panics if not completion) + #[expect( + dead_code, + reason = "Arc accessor is introduced before later stacked PRs use it from completion stages" + )] + #[expect( + clippy::panic, + reason = "typed accessor: caller guarantees variant via RequestType construction" + )] + pub fn completion_request_arc(&self) -> Arc { + match &self.input.request_type { + RequestType::Completion(req) => Arc::clone(req), + _ => panic!("Expected completion request"), + } + } + /// Get Arc clone of responses request (panics if not responses) #[expect( clippy::panic, @@ -419,6 +485,7 @@ impl RequestContext { match &self.input.request_type { RequestType::Chat(req) => req.stream, RequestType::Generate(req) => req.stream, + RequestType::Completion(req) => req.stream, RequestType::Responses(req) => req.stream.unwrap_or(false), RequestType::Messages(req) => req.stream.unwrap_or(false), RequestType::Embedding(_) => false, // Embeddings are never streaming @@ -597,10 +664,16 @@ pub(crate) enum ExecutionResult { /// Final processed response #[derive(Debug)] +#[expect( + dead_code, + reason = "Completion responses are typed in the pipeline before a later stage constructs them" +)] pub(crate) enum FinalResponse { Chat(ChatCompletionResponse), /// Generate response is a Vec of GenerateResponse (n=1 returns single item, n>1 returns multiple) Generate(Vec), + /// Completion response (OpenAI /v1/completions format) + Completion(CompletionResponse), /// Embedding response Embedding(EmbeddingResponse), /// Classification response diff --git a/model_gateway/src/routers/grpc/harmony/stages/request_building.rs b/model_gateway/src/routers/grpc/harmony/stages/request_building.rs index 0ff5108450..1b1630fe9e 100644 --- a/model_gateway/src/routers/grpc/harmony/stages/request_building.rs +++ b/model_gateway/src/routers/grpc/harmony/stages/request_building.rs @@ -63,6 +63,7 @@ impl PipelineStage for HarmonyRequestBuildingStage { RequestType::Chat(_) => format!("chatcmpl-{}", Uuid::now_v7()), RequestType::Responses(_) => format!("responses-{}", Uuid::now_v7()), request_type @ (RequestType::Generate(_) + | RequestType::Completion(_) | RequestType::Embedding(_) | RequestType::Classify(_) | RequestType::Messages(_)) => { diff --git a/model_gateway/src/routers/grpc/harmony/stages/response_processing.rs b/model_gateway/src/routers/grpc/harmony/stages/response_processing.rs index 6a6d6961eb..4e574b66a5 100644 --- a/model_gateway/src/routers/grpc/harmony/stages/response_processing.rs +++ b/model_gateway/src/routers/grpc/harmony/stages/response_processing.rs @@ -139,6 +139,7 @@ impl PipelineStage for HarmonyResponseProcessingStage { Ok(None) } request_type @ (RequestType::Generate(_) + | RequestType::Completion(_) | RequestType::Embedding(_) | RequestType::Classify(_) | RequestType::Messages(_)) => { diff --git a/model_gateway/src/routers/grpc/pipeline.rs b/model_gateway/src/routers/grpc/pipeline.rs index 86d67ec114..181c13a984 100644 --- a/model_gateway/src/routers/grpc/pipeline.rs +++ b/model_gateway/src/routers/grpc/pipeline.rs @@ -9,6 +9,7 @@ use axum::response::{IntoResponse, Response}; use openai_protocol::{ chat::{ChatCompletionRequest, ChatCompletionResponse}, classify::ClassifyRequest, + completion::CompletionRequest, embedding::EmbeddingRequest, generate::GenerateRequest, messages::CreateMessageRequest, @@ -60,6 +61,48 @@ pub(crate) struct RequestPipeline { } impl RequestPipeline { + fn wrong_response_type( + &self, + function: &'static str, + expected: &'static str, + response_type: &FinalResponse, + model: &str, + endpoint: &'static str, + ) -> Response { + error!( + function = function, + response_type = %response_type, + "Wrong response type: expected {expected}, got {response_type}" + ); + Metrics::record_router_error( + metrics_labels::ROUTER_GRPC, + self.backend_type, + metrics_labels::CONNECTION_GRPC, + model, + endpoint, + metrics_labels::ERROR_INTERNAL, + ); + error::internal_error("wrong_response_type", "Internal error: wrong response type") + } + + fn no_response_produced( + &self, + function: &'static str, + model: &str, + endpoint: &'static str, + ) -> Response { + error!(function = function, "No response produced by pipeline"); + Metrics::record_router_error( + metrics_labels::ROUTER_GRPC, + self.backend_type, + metrics_labels::CONNECTION_GRPC, + model, + endpoint, + metrics_labels::ERROR_INTERNAL, + ); + error::internal_error("no_response_produced", "No response produced") + } + /// Create a regular (single-worker) pipeline pub fn new_regular( worker_registry: Arc, @@ -429,39 +472,24 @@ impl RequestPipeline { ); axum::Json(response).into_response() } - Some(FinalResponse::Generate(_)) - | Some(FinalResponse::Embedding(_)) - | Some(FinalResponse::Classify(_)) - | Some(FinalResponse::Messages(_)) => { - error!( - function = "execute_chat", - "Wrong response type: expected Chat, got Generate/Embedding/Classify/Messages" - ); - Metrics::record_router_error( - metrics_labels::ROUTER_GRPC, - self.backend_type, - metrics_labels::CONNECTION_GRPC, - &request_for_metrics.model, - metrics_labels::ENDPOINT_CHAT, - metrics_labels::ERROR_INTERNAL, - ); - error::internal_error("wrong_response_type", "Internal error: wrong response type") - } - None => { - error!( - function = "execute_chat", - "No response produced by pipeline" - ); - Metrics::record_router_error( - metrics_labels::ROUTER_GRPC, - self.backend_type, - metrics_labels::CONNECTION_GRPC, - &request_for_metrics.model, - metrics_labels::ENDPOINT_CHAT, - metrics_labels::ERROR_INTERNAL, - ); - error::internal_error("no_response_produced", "No response produced") - } + Some( + response_type @ (FinalResponse::Generate(_) + | FinalResponse::Completion(_) + | FinalResponse::Embedding(_) + | FinalResponse::Classify(_) + | FinalResponse::Messages(_)), + ) => self.wrong_response_type( + "execute_chat", + "Chat", + &response_type, + &request_for_metrics.model, + metrics_labels::ENDPOINT_CHAT, + ), + None => self.no_response_produced( + "execute_chat", + &request_for_metrics.model, + metrics_labels::ENDPOINT_CHAT, + ), } } @@ -533,39 +561,117 @@ impl RequestPipeline { ); axum::Json(response).into_response() } - Some(FinalResponse::Chat(_)) - | Some(FinalResponse::Embedding(_)) - | Some(FinalResponse::Classify(_)) - | Some(FinalResponse::Messages(_)) => { - error!( - function = "execute_generate", - "Wrong response type: expected Generate, got Chat/Embedding/Classify/Messages" - ); - Metrics::record_router_error( - metrics_labels::ROUTER_GRPC, - self.backend_type, - metrics_labels::CONNECTION_GRPC, - &model_id, - metrics_labels::ENDPOINT_GENERATE, - metrics_labels::ERROR_INTERNAL, - ); - error::internal_error("wrong_response_type", "Internal error: wrong response type") + Some( + response_type @ (FinalResponse::Chat(_) + | FinalResponse::Completion(_) + | FinalResponse::Embedding(_) + | FinalResponse::Classify(_) + | FinalResponse::Messages(_)), + ) => self.wrong_response_type( + "execute_generate", + "Generate", + &response_type, + &model_id, + metrics_labels::ENDPOINT_GENERATE, + ), + None => self.no_response_produced( + "execute_generate", + &model_id, + metrics_labels::ENDPOINT_GENERATE, + ), + } + } + + /// Execute the complete pipeline for a completion request + #[expect( + dead_code, + reason = "Completion pipeline entrypoint is introduced before later stacked PRs wire the router to call it" + )] + pub async fn execute_completion( + &self, + request: Arc, + headers: Option, + _model_id: Option, + components: Arc, + ) -> Response { + let start = Instant::now(); + let model = request.model.clone(); + let streaming = request.stream; + + Metrics::record_router_request( + metrics_labels::ROUTER_GRPC, + self.backend_type, + metrics_labels::CONNECTION_GRPC, + &model, + metrics_labels::ENDPOINT_COMPLETIONS, + bool_to_static_str(streaming), + ); + + let mut ctx = RequestContext::for_completion(request, headers, model.clone(), components); + + for stage in self.stages.iter() { + match stage.execute(&mut ctx).await { + Ok(Some(response)) => { + Metrics::record_router_duration( + metrics_labels::ROUTER_GRPC, + self.backend_type, + metrics_labels::CONNECTION_GRPC, + &model, + metrics_labels::ENDPOINT_COMPLETIONS, + start.elapsed(), + ); + return response; + } + Ok(None) => continue, + Err(response) => { + Metrics::record_router_error( + metrics_labels::ROUTER_GRPC, + self.backend_type, + metrics_labels::CONNECTION_GRPC, + &model, + metrics_labels::ENDPOINT_COMPLETIONS, + error_type_from_status(response.status()), + ); + error!( + "Stage {} failed with status {}", + stage.name(), + response.status() + ); + return response; + } } - None => { - error!( - function = "execute_generate", - "No response produced by pipeline" - ); - Metrics::record_router_error( + } + + match ctx.state.response.final_response { + Some(FinalResponse::Completion(response)) => { + Metrics::record_router_duration( metrics_labels::ROUTER_GRPC, self.backend_type, metrics_labels::CONNECTION_GRPC, - &model_id, - metrics_labels::ENDPOINT_GENERATE, - metrics_labels::ERROR_INTERNAL, + &model, + metrics_labels::ENDPOINT_COMPLETIONS, + start.elapsed(), ); - error::internal_error("no_response_produced", "No response produced") + axum::Json(response).into_response() } + Some( + response_type @ (FinalResponse::Chat(_) + | FinalResponse::Generate(_) + | FinalResponse::Embedding(_) + | FinalResponse::Classify(_) + | FinalResponse::Messages(_)), + ) => self.wrong_response_type( + "execute_completion", + "Completion", + &response_type, + &model, + metrics_labels::ENDPOINT_COMPLETIONS, + ), + None => self.no_response_produced( + "execute_completion", + &model, + metrics_labels::ENDPOINT_COMPLETIONS, + ), } } @@ -838,39 +944,24 @@ impl RequestPipeline { ); axum::Json(response).into_response() } - Some(FinalResponse::Chat(_)) - | Some(FinalResponse::Generate(_)) - | Some(FinalResponse::Embedding(_)) - | Some(FinalResponse::Classify(_)) => { - error!( - function = "execute_messages", - "Wrong response type: expected Messages" - ); - Metrics::record_router_error( - metrics_labels::ROUTER_GRPC, - self.backend_type, - metrics_labels::CONNECTION_GRPC, - &request.model, - metrics_labels::ENDPOINT_MESSAGES, - metrics_labels::ERROR_INTERNAL, - ); - error::internal_error("wrong_response_type", "Internal error: wrong response type") - } - None => { - error!( - function = "execute_messages", - "No response produced by pipeline" - ); - Metrics::record_router_error( - metrics_labels::ROUTER_GRPC, - self.backend_type, - metrics_labels::CONNECTION_GRPC, - &request.model, - metrics_labels::ENDPOINT_MESSAGES, - metrics_labels::ERROR_INTERNAL, - ); - error::internal_error("no_response_produced", "No response produced") - } + Some( + response_type @ (FinalResponse::Chat(_) + | FinalResponse::Generate(_) + | FinalResponse::Completion(_) + | FinalResponse::Embedding(_) + | FinalResponse::Classify(_)), + ) => self.wrong_response_type( + "execute_messages", + "Messages", + &response_type, + &request.model, + metrics_labels::ENDPOINT_MESSAGES, + ), + None => self.no_response_produced( + "execute_messages", + &request.model, + metrics_labels::ENDPOINT_MESSAGES, + ), } } @@ -921,6 +1012,7 @@ impl RequestPipeline { match ctx.state.response.final_response { Some(FinalResponse::Chat(response)) => Ok(response), Some(FinalResponse::Generate(_)) + | Some(FinalResponse::Completion(_)) | Some(FinalResponse::Embedding(_)) | Some(FinalResponse::Classify(_)) | Some(FinalResponse::Messages(_)) => { diff --git a/model_gateway/src/routers/grpc/regular/stages/request_building.rs b/model_gateway/src/routers/grpc/regular/stages/request_building.rs index c478963cae..397502593b 100644 --- a/model_gateway/src/routers/grpc/regular/stages/request_building.rs +++ b/model_gateway/src/routers/grpc/regular/stages/request_building.rs @@ -41,7 +41,9 @@ impl PipelineStage for RequestBuildingStage { RequestType::Generate(_) => self.generate_stage.execute(ctx).await, RequestType::Embedding(_) => self.embedding_stage.execute(ctx).await, RequestType::Classify(_) => self.embedding_stage.execute(ctx).await, - request_type @ (RequestType::Responses(_) | RequestType::Messages(_)) => { + request_type @ (RequestType::Completion(_) + | RequestType::Responses(_) + | RequestType::Messages(_)) => { error!( function = "RequestBuildingStage::execute", request_type = %request_type, diff --git a/model_gateway/src/routers/grpc/regular/stages/response_processing.rs b/model_gateway/src/routers/grpc/regular/stages/response_processing.rs index 2bd6f51a5f..88ab6408e6 100644 --- a/model_gateway/src/routers/grpc/regular/stages/response_processing.rs +++ b/model_gateway/src/routers/grpc/regular/stages/response_processing.rs @@ -53,7 +53,9 @@ impl PipelineStage for ResponseProcessingStage { RequestType::Generate(_) => self.generate_stage.execute(ctx).await, RequestType::Embedding(_) => self.embedding_stage.execute(ctx).await, RequestType::Classify(_) => self.classify_stage.execute(ctx).await, - request_type @ (RequestType::Responses(_) | RequestType::Messages(_)) => { + request_type @ (RequestType::Completion(_) + | RequestType::Responses(_) + | RequestType::Messages(_)) => { error!( function = "ResponseProcessingStage::execute", request_type = %request_type, diff --git a/model_gateway/src/server.rs b/model_gateway/src/server.rs index cb0a60ddbe..a80733dfe1 100644 --- a/model_gateway/src/server.rs +++ b/model_gateway/src/server.rs @@ -200,7 +200,7 @@ async fn v1_chat_completions( async fn v1_completions( State(state): State>, headers: http::HeaderMap, - Json(body): Json, + ValidatedJson(body): ValidatedJson, ) -> Response { state .router