diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 75acf87a7d89..5dafcbe13c25 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -1,5 +1,5 @@ import { InfoCircleOutlined } from "@ant-design/icons"; -import { Select as AntdSelect, Card, InputNumber, Radio, Space, Tooltip, Typography } from "antd"; +import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Tooltip, Typography } from "antd"; import React from "react"; import { ClassifierType, @@ -47,6 +47,8 @@ const ClassificationMethodConfig: React.FC = ({ classifierType === "llm" ? value.classifier_context_per_turn_chars ?? DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS : undefined, + classifier_context_include_assistant_turns: + classifierType === "llm" ? value.classifier_context_include_assistant_turns : undefined, }; onChange(nextValue); }; @@ -85,6 +87,13 @@ const ClassificationMethodConfig: React.FC = ({ }); }; + const handleClassifierContextIncludeAssistantTurnsChange = (includeAssistantTurns: boolean) => { + onChange({ + ...value, + classifier_context_include_assistant_turns: includeAssistantTurns, + }); + }; + return ( <> = ({ Prior turns longer than this are truncated. +
+
+ + Include Assistant Turns + + + +
+ + Let the classifier read the assistant's replies, so difficulty the model stated rather than the user + stays visible: a plan the assistant calls complex, approved with "yes", is classified on the + work being approved. Context Window Size then counts the last N turns across both roles rather than the + last N user turns. + +
)} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index a1e992bd46b4..319cc7b6fbff 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -147,6 +147,57 @@ describe("ComplexityRouterConfig", () => { expect(within(perTurnCharsSection).getByDisplayValue("200")).toBeInTheDocument(); }); + it("should show the assistant-turns switch with its configured value when classifier_type is llm", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 750 }, + classifier_context_include_assistant_turns: true, + }; + renderWithProviders(); + + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + expect(screen.getByText("Include Assistant Turns")).toBeInTheDocument(); + expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).toBeChecked(); + }); + + it("should render the assistant-turns switch off when it is not set", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + renderWithProviders(); + + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).not.toBeChecked(); + }); + + it("should hide the assistant-turns switch when classifier_type is heuristic", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.queryByText("Include Assistant Turns")).not.toBeInTheDocument(); + }); + + it("should call onChange when the assistant-turns switch is toggled", () => { + const onChange = vi.fn(); + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + renderWithProviders(); + + fireEvent.click(screen.getByText("Advanced: Classification Method")); + fireEvent.click(screen.getByRole("switch", { name: "Include Assistant Turns" })); + + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ classifier_context_include_assistant_turns: true }), + ); + }); + it("should hide classifier context fields when classifier_type is heuristic", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Classification Method")); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index de32d15d5a79..0503c0c9c6d5 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -44,6 +44,7 @@ export interface ComplexityRouterConfigValue { classifier_llm_config?: ClassifierLLMConfig; classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; + classifier_context_include_assistant_turns?: boolean; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; tier_distance_penalty?: number; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index fb7eabd7110a..4593f6a6a2ed 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -101,6 +101,7 @@ const AddAutoRouterTab: React.FC = ({ classifier_llm_config: classifierLlmConfig, classifier_context_window_size: classifierContextWindowSize, classifier_context_per_turn_chars: classifierContextPerTurnChars, + classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, adaptive = false, adaptive_weights: adaptiveWeights = DEFAULT_ADAPTIVE_WEIGHTS, tier_distance_penalty: tierDistancePenalty = DEFAULT_TIER_DISTANCE_PENALTY, @@ -146,6 +147,7 @@ const AddAutoRouterTab: React.FC = ({ classifierLlmConfig, classifierContextWindowSize, classifierContextPerTurnChars, + classifierContextIncludeAssistantTurns, customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index e269a3c9028d..e939ce129047 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -18,6 +18,7 @@ const baseParams: BuildComplexityRouterConfigParams = { classifierLlmConfig: undefined, classifierContextWindowSize: undefined, classifierContextPerTurnChars: undefined, + classifierContextIncludeAssistantTurns: undefined, customTechnicalKeywords: [], keywordTierRules: [], semanticMatchingEnabled: false, @@ -318,3 +319,35 @@ describe("getSemanticConfigError", () => { ).toBeNull(); }); }); + +describe("buildComplexityRouterConfig assistant turns", () => { + const llmParams: BuildComplexityRouterConfigParams = { + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + }; + + it("emits the field when the LLM classifier is selected", () => { + const config = buildComplexityRouterConfig({ ...llmParams, classifierContextIncludeAssistantTurns: true }); + expect(config.classifier_context_include_assistant_turns).toBe(true); + }); + + it("emits the switch turned off, since false is a choice the operator made and not an absent value", () => { + const config = buildComplexityRouterConfig({ ...llmParams, classifierContextIncludeAssistantTurns: false }); + expect(config.classifier_context_include_assistant_turns).toBe(false); + }); + + it("omits it when classifier_type is heuristic even if a value lingers in state", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "heuristic", + classifierContextIncludeAssistantTurns: true, + }); + expect(config.classifier_context_include_assistant_turns).toBeUndefined(); + }); + + it("omits it when unset, leaving the backend default", () => { + const config = buildComplexityRouterConfig(llmParams); + expect(config.classifier_context_include_assistant_turns).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 04a9b0f9bd47..192e71b4597a 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -14,6 +14,7 @@ export interface BuildComplexityRouterConfigParams { classifierLlmConfig: ClassifierLLMConfig | undefined; classifierContextWindowSize: number | undefined; classifierContextPerTurnChars: number | undefined; + classifierContextIncludeAssistantTurns: boolean | undefined; customTechnicalKeywords: string[]; keywordTierRules: KeywordTierRule[]; semanticMatchingEnabled: boolean; @@ -33,6 +34,7 @@ export interface ComplexityRouterConfigPayload { classifier_llm_config?: ClassifierLLMConfig; classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; + classifier_context_include_assistant_turns?: boolean; custom_technical_keywords?: string[]; keyword_tier_rules?: { keywords: string[]; tier: KeywordTierRule["tier"] }[]; semantic_keyword_matching?: boolean; @@ -75,6 +77,7 @@ export const buildComplexityRouterConfig = ({ classifierLlmConfig, classifierContextWindowSize, classifierContextPerTurnChars, + classifierContextIncludeAssistantTurns, customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, @@ -103,6 +106,10 @@ export const buildComplexityRouterConfig = ({ classifierContextPerTurnChars !== undefined && { classifier_context_per_turn_chars: classifierContextPerTurnChars, }), + ...(classifierType === "llm" && + classifierContextIncludeAssistantTurns !== undefined && { + classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns, + }), ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), escalation_keywords: cleanedEscalationKeywords, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index eef5e1e4d06f..f8d46f9ddd68 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -151,3 +151,51 @@ describe("buildUpdatedComplexityRouterConfig classifier context window", () => { expect(result.classifier_context_per_turn_chars).toBeUndefined(); }); }); + +const STORED_ASSISTANT_CTX = { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 }, + classifier_context_include_assistant_turns: true, +}; + +describe("buildUpdatedComplexityRouterConfig assistant turns", () => { + const formBase = { + tiers: STORED_ASSISTANT_CTX.tiers, + classifier_type: "llm" as const, + classifier_llm_config: STORED_ASSISTANT_CTX.classifier_llm_config, + }; + + it("round-trips an untouched edit without changing the value", () => { + const result = buildUpdatedComplexityRouterConfig(STORED_ASSISTANT_CTX, { + ...formBase, + classifier_context_include_assistant_turns: true, + }); + expect(result.classifier_context_include_assistant_turns).toBe(true); + }); + + it("persists turning assistant turns back off", () => { + // The off case is the one a preserved-config fallback would silently lose, since false and + // "absent" look alike to a truthiness check. + const result = buildUpdatedComplexityRouterConfig(STORED_ASSISTANT_CTX, { + ...formBase, + classifier_context_include_assistant_turns: false, + }); + expect(result.classifier_context_include_assistant_turns).toBe(false); + }); + + it("omits it when classifier_type is heuristic even if a value lingers in state", () => { + const result = buildUpdatedComplexityRouterConfig(STORED_ASSISTANT_CTX, { + tiers: STORED_ASSISTANT_CTX.tiers, + classifier_type: "heuristic" as const, + classifier_context_include_assistant_turns: true, + }); + expect(result.classifier_context_include_assistant_turns).toBeUndefined(); + }); + + it("does not resurrect a stale stored value once the form's own value is unset", () => { + // A MANAGED key: the form wins over the stored config, never falls back to it. + const result = buildUpdatedComplexityRouterConfig(STORED_ASSISTANT_CTX, formBase); + expect(result.classifier_context_include_assistant_turns).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index 504234e89777..98b7ac519f55 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -184,3 +184,62 @@ describe("EditAutoRouterModal classifier context window", () => { expect(savedConfig().classifier_context_window_size).toBe(8); }); }); + +describe("EditAutoRouterModal assistant turns", () => { + beforeEach(() => { + modelPatchUpdateCall.mockClear(); + }); + + const STORED_CONFIG = { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o-mini"], COMPLEX: ["gpt-4o-mini"], REASONING: ["gpt-4o-mini"] }, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 }, + classifier_context_include_assistant_turns: true, + }; + + const renderModal = () => + renderWithProviders( + , + ); + + // The create and edit stacks share the rendered control but duplicate the serializer, the + // hydrator and the managed-key set, so a field wired into only one of them fails here and + // nowhere else: the payload-builder unit tests are handed a form value assembled by hand. + it("shows the stored value and preserves it through an untouched open-and-save", async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await screen.findByText("Include Assistant Turns"); + expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classifier_context_include_assistant_turns).toBe(true); + }); + + it("persists turning assistant turns off", async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await screen.findByText("Include Assistant Turns"); + await user.click(screen.getByRole("switch", { name: "Include Assistant Turns" })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classifier_context_include_assistant_turns).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 2686f99307e6..99b5ff178b12 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -35,6 +35,7 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_llm_config", "classifier_context_window_size", "classifier_context_per_turn_chars", + "classifier_context_include_assistant_turns", "adaptive", "adaptive_weights", "tier_distance_penalty", @@ -96,6 +97,10 @@ export const buildUpdatedComplexityRouterConfig = ( value.classifier_context_per_turn_chars !== undefined && { classifier_context_per_turn_chars: value.classifier_context_per_turn_chars, }), + ...(value.classifier_type === "llm" && + value.classifier_context_include_assistant_turns !== undefined && { + classifier_context_include_assistant_turns: value.classifier_context_include_assistant_turns, + }), ...(customTechnicalKeywords && customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords, @@ -209,6 +214,10 @@ const EditAutoRouterModal: React.FC = ({ typeof parsedConfig.classifier_context_per_turn_chars === "number" ? parsedConfig.classifier_context_per_turn_chars : undefined, + classifier_context_include_assistant_turns: + typeof parsedConfig.classifier_context_include_assistant_turns === "boolean" + ? parsedConfig.classifier_context_include_assistant_turns + : undefined, adaptive: parsedConfig.adaptive || false, adaptive_weights: parsedConfig.adaptive_weights, tier_distance_penalty: parsedConfig.tier_distance_penalty,