Allow 128 chat logprob candidates through the model gateway

sgl-model-gateway/Cargo.toml

@@ -205,3 +205,6 @@ codegen-units = 1
 [profile.dev-opt]
 inherits = "dev"
 opt-level = 1
+
+[patch.crates-io]
+openai-protocol = { path = "vendor/openai-protocol" }

sgl-model-gateway/src/chat_protocol_tests.rs

@@ -0,0 +1,78 @@
+#[cfg(test)]
+mod tests {
+    use openai_protocol::{chat::ChatCompletionRequest, validated::Normalizable};
+    use validator::Validate;
+    use serde_json::json;
+
+    fn request(top_logprobs: u32) -> ChatCompletionRequest {
+        serde_json::from_value(json!({
+            "model": "test",
+            "messages": [{"role": "user", "content": "hello"}],
+            "logprobs": true,
+            "top_logprobs": top_logprobs
+        }))
+        .unwrap()
+    }
+
+    #[test]
+    fn accepts_extended_head_without_modifying_request() {
+        for count in [0, 16, 20, 21, 128] {
+            let mut body = request(count);
+            body.normalize();
+            assert!(body.validate().is_ok(), "{count}");
+            assert_eq!(
+                serde_json::to_value(&body).unwrap()["top_logprobs"],
+                count
+            );
+        }
+    }
+
+    #[test]
+    fn rejects_head_above_backend_limit() {
+        for count in [129, u32::MAX] {
+            let errors = request(count).validate().unwrap_err();
+            assert!(errors.field_errors().contains_key("top_logprobs"));
+        }
+    }
+
+    #[test]
+    fn preserves_other_protocol_validation() {
+        let mut body = request(128);
+        body.temperature = Some(3.0);
+        let errors = body.validate().unwrap_err();
+        assert!(errors.field_errors().contains_key("temperature"));
+    }
+
+    #[test]
+    fn optional_head_and_invalid_json_types() {
+        let value = json!({"model":"test","messages":[{"role":"user","content":"hello"}]});
+        let body: ChatCompletionRequest = serde_json::from_value(value.clone()).unwrap();
+        assert!(body.validate().is_ok());
+        for bad in [json!(-1), json!(1.5), json!("128")] {
+            let mut value = value.clone();
+            value["top_logprobs"] = bad;
+            assert!(serde_json::from_value::<ChatCompletionRequest>(value).is_err());
+        }
+    }
+    #[test]
+    fn preserves_session_extensions() {
+        let mut value = serde_json::to_value(request(128)).unwrap();
+        value["input_ids"] = json!([1, 2, 3]);
+        value["return_meta_info"] = json!(true);
+        let mut body: ChatCompletionRequest = serde_json::from_value(value).unwrap();
+        body.normalize();
+        assert!(body.validate().is_ok());
+        let outgoing = serde_json::to_value(body).unwrap();
+        assert_eq!(outgoing["input_ids"], json!([1, 2, 3]));
+        assert_eq!(outgoing["return_meta_info"], json!(true));
+        assert_eq!(outgoing["top_logprobs"], json!(128));
+    }
+
+    #[test]
+    fn requires_logprobs_for_extended_head() {
+        let mut body = request(128);
+        body.logprobs = false;
+        assert!(body.validate().is_err());
+    }
+
+}

sgl-model-gateway/src/lib.rs

@@ -1,3 +1,5 @@
+#[cfg(test)]
+mod chat_protocol_tests;
 pub mod app_context;
 pub use smg_auth as auth;
 pub mod config;

sgl-model-gateway/vendor/README.md

@@ -0,0 +1 @@
+openai-protocol 1.0.0 is vendored from crates.io under Apache-2.0. Local changes extend the chat top_logprobs maximum to128 and preserve input_ids and return_meta_info so session clients retain exact tokenization and replay metadata. Validation remains in the schema, including cross-field checks. All other protocol fields remain unchanged.

sgl-model-gateway/vendor/openai-protocol/Cargo.toml

@@ -0,0 +1,77 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2021"
+name = "openai-protocol"
+version = "1.0.0"
+authors = [
+    "Simo Lin <linsimo.mark@gmail.com>",
+    "Chang Su <mckvtl@gmail.com>",
+    "Keyang Ru <rukeyang@gmail.com>",
+]
+build = false
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
+description = "OpenAI-compatible API protocol definitions and types"
+readme = false
+keywords = [
+    "openai",
+    "llm",
+    "api",
+    "protocol",
+    "types",
+]
+categories = [
+    "api-bindings",
+    "data-structures",
+]
+license = "Apache-2.0"
+repository = "https://github.com/lightseekorg/smg"
+
+[features]
+axum = ["dep:axum"]
+default = []
+
+[lib]
+name = "openai_protocol"
+path = "src/lib.rs"
+
+[dependencies.axum]
+version = "0.8.6"
+optional = true
+
+[dependencies.chrono]
+version = "0.4"
+
+[dependencies.rand]
+version = "0.9.2"
+
+[dependencies.serde]
+version = "1.0"
+features = ["derive"]
+
+[dependencies.serde_json]
+version = "1.0"
+features = ["preserve_order"]
+
+[dependencies.tokio]
+version = "1.42.0"
+
+[dependencies.tracing]
+version = "0.1"
+
+[dependencies.validator]
+version = "0.20.0"
+features = ["derive"]

sgl-model-gateway/vendor/openai-protocol/src/builders/chat/mod.rs

@@ -0,0 +1,7 @@
+//! Builders for Chat Completion API response types
+
+pub mod response;
+pub mod stream_response;
+
+pub use response::ChatCompletionResponseBuilder;
+pub use stream_response::ChatCompletionStreamResponseBuilder;