Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,8 @@ Chat completion and completion requests currently capture extra top-level fields
map named `unsupported_fields`. The name describes the capture mechanism, not the final admission
decision. A small named set is accepted and validated for downstream handling. Other fields are
rejected by default. When `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` is truthy, those other fields
are ignored and dropped; the switch does not enable general forwarding. See the
are ignored and dropped. Otherwise, fields named in `DYN_OPENAI_FE_IGNORED_FIELDS` are dropped the
same way while the rest are still rejected. Neither setting enables general forwarding. See the
[frontend configuration reference](../../../../reference/components/frontend-configuration.mdx)
for the exact switch behavior.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -876,6 +876,15 @@ are disabled by default. Truthy values are `1`, `true`, `yes`, or `on`, case-ins
backend. Named passthrough fields remain subject to their normal validation and handling.
</ParamField>

<ParamField path="DYN_OPENAI_FE_IGNORED_FIELDS" type="string" default="unset">
A comma-separated list of unknown top-level request fields that `/v1/chat/completions` and
`/v1/completions` ignore and drop instead of rejecting, for example `reasoning_split`. Other
unknown fields are still rejected, unless `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` is truthy:
that switch takes precedence and drops every unknown field. Use this list in its place when
clients send a known set of extension fields. Like that switch, it does not forward fields to a
backend.
</ParamField>

<ParamField path="DYN_DISABLE_FRONTEND_NVEXT" type="boolean" default="false">
When truthy, the frontend drops all request NvExt fields except `cache_salt` on
`/v1/chat/completions`, `/v1/completions`, `/v1/responses`, and `/v1/messages`; ignores the
Expand Down
134 changes: 112 additions & 22 deletions lib/llm/src/protocols/openai/validate.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::{borrow::Cow, fmt::Display, sync::LazyLock};
use std::{borrow::Cow, collections::HashSet, fmt::Display, sync::LazyLock};

use dynamo_protocols::types::{
ChatCompletionTool, ChatCompletionToolType, CreateChatCompletionRequest, FunctionObject,
};
use dynamo_runtime::config::{
env_is_truthy, environment_names::llm::DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS,
env_is_truthy,
environment_names::llm::{
DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS, DYN_OPENAI_FE_IGNORED_FIELDS,
},
};
use serde_json::Value;

Expand Down Expand Up @@ -145,30 +148,62 @@ where
static IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS: LazyLock<bool> =
LazyLock::new(|| env_is_truthy(DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS));

static OPENAI_FE_IGNORED_FIELDS: LazyLock<HashSet<String>> = LazyLock::new(|| {
parse_ignored_fields(&std::env::var(DYN_OPENAI_FE_IGNORED_FIELDS).unwrap_or_default())
});

/// Unknown top-level fields that are dropped instead of rejected.
Comment thread
sumant-TT marked this conversation as resolved.
Resolved
#[derive(Clone, Copy)]
enum IgnoredFields<'a> {
All,
Named(&'a HashSet<String>),
}

impl IgnoredFields<'_> {
fn contains(self, field: &str) -> bool {
match self {
Self::All => true,
Self::Named(fields) => fields.contains(field),
}
}
}

Comment thread
sumant-TT marked this conversation as resolved.
Resolved
/// Parse a comma-separated list of field names. Names are trimmed and empty items dropped.
fn parse_ignored_fields(value: &str) -> HashSet<String> {
value
.split(',')
.map(str::trim)
.filter(|field| !field.is_empty())
.map(str::to_owned)
.collect()
}

/// Validates that no unsupported fields are present in the request.
///
/// Fields in `PASSTHROUGH_EXTRA_FIELDS` are validated by downstream handlers.
/// Other fields may be ignored and dropped when
/// `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` is truthy.
/// Other fields are ignored and dropped when `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS`
/// is truthy, or when they are named in `DYN_OPENAI_FE_IGNORED_FIELDS`.
pub fn validate_no_unsupported_fields(
unsupported_fields: &std::collections::HashMap<String, serde_json::Value>,
) -> Result<(), anyhow::Error> {
validate_no_unsupported_fields_with_ignore(
unsupported_fields,
*IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS,
)
let ignored = if *IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS {
IgnoredFields::All
} else {
IgnoredFields::Named(&OPENAI_FE_IGNORED_FIELDS)
};
validate_no_unsupported_fields_with_ignore(unsupported_fields, ignored)
}

fn validate_no_unsupported_fields_with_ignore(
unsupported_fields: &std::collections::HashMap<String, serde_json::Value>,
ignore_unsupported_fields: bool,
ignored: IgnoredFields<'_>,
) -> Result<(), anyhow::Error> {
let unknown: Vec<_> = unsupported_fields
.keys()
.filter(|k| !PASSTHROUGH_EXTRA_FIELDS.contains(&k.as_str()))
.filter(|k| !PASSTHROUGH_EXTRA_FIELDS.contains(&k.as_str()) && !ignored.contains(k))
.map(|s| format!("`{}`", s))
.collect();
if !unknown.is_empty() && !ignore_unsupported_fields {
if !unknown.is_empty() {
anyhow::bail!("Unsupported parameter(s): {}", unknown.join(", "));
}
if let Some(value) = unsupported_fields.get("cache_salt")
Expand Down Expand Up @@ -1129,7 +1164,7 @@ pub fn validate_chat_only_generation_flags(

#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};

use serde_json::json;

Expand Down Expand Up @@ -1323,7 +1358,11 @@ mod tests {
"vllm_xargs": {"flag": true},
}))
.unwrap();
validate_no_unsupported_fields_with_ignore(&fields.fields, false).unwrap();
validate_no_unsupported_fields_with_ignore(
&fields.fields,
IgnoredFields::Named(&HashSet::new()),
)
.unwrap();
assert_eq!(fields.fields.len(), PASSTHROUGH_EXTRA_FIELDS.len());
assert_eq!(fields.fields["detokenize"], json!(false));
assert_eq!(fields.fields["stop_token_ids"], json!([]));
Expand All @@ -1340,23 +1379,31 @@ mod tests {
] {
let fields: ExtraFields = serde_json::from_value(json!({field: bad})).unwrap();
assert_eq!(fields.fields[field], bad);
let err =
validate_no_unsupported_fields_with_ignore(&fields.fields, false).unwrap_err();
let err = validate_no_unsupported_fields_with_ignore(
&fields.fields,
IgnoredFields::Named(&HashSet::new()),
)
.unwrap_err();
assert!(err.to_string().contains(field), "{err}");
}
}

#[test]
fn validate_no_unsupported_fields_accepts_logprob_token_ids() {
let fields = HashMap::from([("logprob_token_ids".to_string(), json!([14, 15]))]);
validate_no_unsupported_fields_with_ignore(&fields, false).unwrap();
validate_no_unsupported_fields_with_ignore(&fields, IgnoredFields::Named(&HashSet::new()))
.unwrap();
}

#[test]
fn validate_no_unsupported_fields_rejects_malformed_logprob_token_ids() {
for bad in [json!(["notanint"]), json!(7), json!([[1, 2]]), json!([-1])] {
let fields = HashMap::from([("logprob_token_ids".to_string(), bad)]);
let err = validate_no_unsupported_fields_with_ignore(&fields, false).unwrap_err();
let err = validate_no_unsupported_fields_with_ignore(
&fields,
IgnoredFields::Named(&HashSet::new()),
)
.unwrap_err();
assert!(err.to_string().contains("must be an array of token IDs"));
}
}
Expand All @@ -1380,7 +1427,8 @@ mod tests {
"tag": "read",
}),
)]);
validate_no_unsupported_fields_with_ignore(&fields, false).unwrap();
validate_no_unsupported_fields_with_ignore(&fields, IgnoredFields::Named(&HashSet::new()))
.unwrap();
}

#[test]
Expand All @@ -1401,20 +1449,61 @@ mod tests {
),
] {
let fields = HashMap::from([("vllm_xargs".to_string(), bad)]);
let err = validate_no_unsupported_fields_with_ignore(&fields, false).unwrap_err();
let err = validate_no_unsupported_fields_with_ignore(
&fields,
IgnoredFields::Named(&HashSet::new()),
)
.unwrap_err();
assert!(err.to_string().contains(message), "{err}");
}
}

#[test]
fn validate_no_unsupported_fields_rejects_unknown_fields_by_default() {
let err = validate_no_unsupported_fields_with_ignore(&unknown_fields(), false).unwrap_err();
let err = validate_no_unsupported_fields_with_ignore(
&unknown_fields(),
IgnoredFields::Named(&HashSet::new()),
)
.unwrap_err();
assert!(err.to_string().contains("Unsupported parameter(s)"));
}

#[test]
fn validate_no_unsupported_fields_ignores_unknown_fields_when_configured() {
validate_no_unsupported_fields_with_ignore(&unknown_fields(), true).unwrap();
validate_no_unsupported_fields_with_ignore(&unknown_fields(), IgnoredFields::All).unwrap();
}

#[test]
fn validate_no_unsupported_fields_ignores_only_named_fields() {
let named = parse_ignored_fields("experimental_field");
validate_no_unsupported_fields_with_ignore(&unknown_fields(), IgnoredFields::Named(&named))
.unwrap();

let mut fields = unknown_fields();
fields.insert("other_field".to_string(), json!(true));
let err = validate_no_unsupported_fields_with_ignore(&fields, IgnoredFields::Named(&named))
.unwrap_err()
.to_string();
assert!(err.contains("`other_field`"), "{err}");
assert!(!err.contains("experimental_field"), "{err}");
}

#[test]
Comment thread
sumant-TT marked this conversation as resolved.
Resolved
fn validate_no_unsupported_fields_still_validates_named_passthrough_fields() {
let named = parse_ignored_fields("stop_token_ids");
let fields = HashMap::from([("stop_token_ids".to_string(), json!("bad"))]);
let err = validate_no_unsupported_fields_with_ignore(&fields, IgnoredFields::Named(&named))
.unwrap_err();
assert!(err.to_string().contains("stop_token_ids"));
}

#[test]
fn parse_ignored_fields_trims_and_drops_empty_items() {
assert_eq!(
parse_ignored_fields(" reasoning_split, ,other_field,"),
HashSet::from(["reasoning_split".to_string(), "other_field".to_string()])
);
assert!(parse_ignored_fields("").is_empty());
}

#[test]
Expand All @@ -1425,7 +1514,8 @@ mod tests {
]);

let err =
validate_no_unsupported_fields_with_ignore(&unsupported_fields, true).unwrap_err();
validate_no_unsupported_fields_with_ignore(&unsupported_fields, IgnoredFields::All)
.unwrap_err();
assert!(err.to_string().contains("stop_token_ids"));
}

Expand Down
6 changes: 6 additions & 0 deletions lib/runtime/src/config/environment_names.rs
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,11 @@ pub mod llm {
pub const DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS: &str =
"DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS";

/// Comma-separated unknown OpenAI frontend request fields to drop instead of
/// rejecting. Other unknown fields are still rejected unless
/// `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` is truthy.
pub const DYN_OPENAI_FE_IGNORED_FIELDS: &str = "DYN_OPENAI_FE_IGNORED_FIELDS";

/// Master switch for the frontend's HTTP admin API surface.
/// The admin API is **enabled by default**; this variable disables it.
/// Truthy values (`1` / `true` / `yes` / `on`, case-insensitive) prevent
Expand Down Expand Up @@ -1168,6 +1173,7 @@ mod tests {
llm::DYN_ENABLE_ANTHROPIC_API,
llm::DYN_DISABLE_FRONTEND_NVEXT,
llm::DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS,
llm::DYN_OPENAI_FE_IGNORED_FIELDS,
llm::DYN_DISABLE_FRONTEND_ADMIN_API,
llm::DYN_STRIP_ANTHROPIC_PREAMBLE,
llm::DYN_ENABLE_FORCE_INCLUDE_USAGE,
Expand Down
Loading