diff --git a/dashboard/frontend/src/types/config.ts b/dashboard/frontend/src/types/config.ts index 68fe523685e..d2ae4109986 100644 --- a/dashboard/frontend/src/types/config.ts +++ b/dashboard/frontend/src/types/config.ts @@ -268,6 +268,7 @@ export interface DecisionModelSignal { } predicate?: NumericPredicate timeout_ms?: number + prior_user_turns?: number } export interface ComplexityCandidates { diff --git a/e2e/profiles/model-runtime/profile.go b/e2e/profiles/model-runtime/profile.go index ec4e6e0041d..c6567e808cc 100644 --- a/e2e/profiles/model-runtime/profile.go +++ b/e2e/profiles/model-runtime/profile.go @@ -109,6 +109,7 @@ func (p *Profile) GetTestCases() []string { "model-runtime-embeddings-rerank", "decision-runtime-routing", "decision-runtime-set-span", + "decision-prior-user-turns", "model-runtime-bundles", "model-runtime-long-history", "model-runtime-fail-open", diff --git a/e2e/profiles/model-runtime/values.yaml b/e2e/profiles/model-runtime/values.yaml index 2674f579876..473d6320212 100644 --- a/e2e/profiles/model-runtime/values.yaml +++ b/e2e/profiles/model-runtime/values.yaml @@ -106,6 +106,25 @@ config: - key: chat description: Anything else timeout_ms: 10000 + # testcases/decision_prior_user_turns.go: reads the previous user turn + # too, so a conversation's follow-up is a separate call. + - name: follow_up_kind + description: Which kind of work the latest turn continues. + deployment: decision-fixture + prior_user_turns: 1 + question: + type: choice + instructions: What kind of work does the latest user turn ask for? + choices: + - key: code + description: Writing, reviewing or debugging code + - key: math + description: Mathematics or quantitative reasoning + - key: writing + description: Prose, poems or other writing + - key: chat + description: Anything else + timeout_ms: 10000 - name: attached_answered description: Matches whenever the attached decision model answers. deployment: attached-decisions @@ -254,6 +273,10 @@ config: - {type: decision, name: request_kind, label: code} - {type: decision, name: request_kind, label: math} - {type: decision, name: request_kind, label: chat} + - {type: decision, name: follow_up_kind, label: code} + - {type: decision, name: follow_up_kind, label: math} + - {type: decision, name: follow_up_kind, label: writing} + - {type: decision, name: follow_up_kind, label: chat} - {type: decision, name: topics, label: billing} - {type: decision, name: topics, label: shipping} - {type: decision, name: names, label: city} diff --git a/e2e/testcases/decision_prior_user_turns.go b/e2e/testcases/decision_prior_user_turns.go new file mode 100644 index 00000000000..de9b13863e7 --- /dev/null +++ b/e2e/testcases/decision_prior_user_turns.go @@ -0,0 +1,107 @@ +package testcases + +import ( + "context" + "fmt" + "net/http" + + "k8s.io/client-go/kubernetes" + + "github.com/vllm-project/semantic-router/e2e/pkg/modelruntime" + pkgtestcases "github.com/vllm-project/semantic-router/e2e/pkg/testcases" +) + +func init() { + pkgtestcases.Register("decision-prior-user-turns", pkgtestcases.TestCase{ + Description: "A decision signal with prior_user_turns routes a follow-up on the runtime's answer for the earlier user turn and the follow-up together", + Tags: []string{"decision", "model-runtime", "routing", "history"}, + Fn: testDecisionPriorUserTurns, + }) +} + +// The follow_up_kind question of e2e/profiles/model-runtime/values.yaml. +var decisionFollowUpQuestion = map[string]modelruntime.Question{ + "follow_up_kind": { + Type: "choice", + Instructions: "What kind of work does the latest user turn ask for?", + Choices: []modelruntime.Choice{ + {Key: "code", Description: "Writing, reviewing or debugging code"}, + {Key: "math", Description: "Mathematics or quantitative reasoning"}, + {Key: "writing", Description: "Prose, poems or other writing"}, + {Key: "chat", Description: "Anything else"}, + }, + }, +} + +const decisionFollowUp = "Now make it shorter." + +var decisionFirstTurns = []string{ + "Write a Go function that merges two sorted slices.", + "Prove that the square root of two is irrational.", + "Write a short poem about rain on a tin roof.", + "Explain how a hash map handles collisions.", + "Draft an email asking my landlord to fix the heating.", + "Solve x squared minus five x plus six equals zero.", +} + +// testDecisionPriorUserTurns finds a first turn for which the fixture runtime +// answers the follow-up differently alone and after that turn, so the route +// shows which state the Router asked. The fixture weights are random, so the +// runtime's own answers are the expectation. +func testDecisionPriorUserTurns(ctx context.Context, client *kubernetes.Clientset, opts pkgtestcases.TestCaseOptions) error { + session, err := openModelRuntimeSession(ctx, client, opts) + if err != nil { + return err + } + defer session.Close() + if err = session.waitReady(ctx, mrDecisionDeployment); err != nil { + return err + } + managed, _, err := session.managed(ctx, mrDecisionDeployment) + if err != nil { + return err + } + for _, first := range decisionFirstTurns { + alone, err := followUpChoice(ctx, managed, decisionFollowUp) + if err != nil { + return err + } + withPrior, err := followUpChoice(ctx, managed, first+"\n\n"+decisionFollowUp) + if err != nil { + return err + } + if alone == withPrior { + continue + } + messages := []map[string]string{ + {"role": "user", "content": first}, + {"role": "assistant", "content": "Here it is."}, + {"role": "user", "content": decisionFollowUp}, + } + response, err := sendLocalChatConversation(ctx, session.gatewayPort, "vllm-sr/auto", messages, mrRequestTimeout) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("follow-up chat: %s", formatUnexpectedChatCompletionStatus(response)) + } + matched := headerItems(response.Headers, "x-vsr-matched-decision-model") + if !matched["follow_up_kind:"+withPrior] || matched["follow_up_kind:"+alone] { + return fmt.Errorf("x-vsr-matched-decision-model %v: want follow_up_kind:%s (the runtime's answer after %q), not follow_up_kind:%s (the follow-up alone)", + matched, withPrior, first, alone) + } + if opts.SetDetails != nil { + opts.SetDetails(map[string]interface{}{"first_turn": first, "alone": alone, "with_prior_turn": withPrior}) + } + return nil + } + return fmt.Errorf("the fixture answers %q the same alone and after each of %d first turns, so no route can show which state was asked", decisionFollowUp, len(decisionFirstTurns)) +} + +func followUpChoice(ctx context.Context, runtime *modelruntime.Client, state string) (string, error) { + answers, err := runtime.Decide(ctx, modelruntime.DecisionsRequest{Model: mrDecisionDeployment, State: state, Questions: decisionFollowUpQuestion}) + if err != nil { + return "", err + } + return answers.Answers["follow_up_kind"].Choice, nil +} diff --git a/src/semantic-router/pkg/classification/classifier_signal_decision_model.go b/src/semantic-router/pkg/classification/classifier_signal_decision_model.go index 8c3fa24afde..df4450abf0e 100644 --- a/src/semantic-router/pkg/classification/classifier_signal_decision_model.go +++ b/src/semantic-router/pkg/classification/classifier_signal_decision_model.go @@ -3,6 +3,7 @@ package classification import ( "context" "slices" + "strings" "sync" "time" @@ -27,46 +28,71 @@ func (c *Classifier) decider() modelservice.Decider { return modelservice.Default() } +// decisionCall is one call's deployment and the earlier user turns its +// questions read. +type decisionCall struct { + deployment string + priorTurns int +} + // evaluateDecisionModelSignals asks every used decision question in one call // per deployment, with the request text as the state. Calls to different // deployments run in parallel. A question that names no deployment asks the // decision model, with the request as it came, like the built-in signals -// whose call it joins. A failed or late call leaves its signals unknown; it -// never fails the request. +// whose call it joins. Questions that read earlier user turns share a call +// only with questions that read as many. A failed or late call leaves its +// signals unknown; it never fails the request. func (c *Classifier) evaluateDecisionModelSignals( ctx context.Context, results *SignalResults, mu *sync.Mutex, text string, wholeText string, + priorUserMessages []string, usedSignals map[string]bool, ) { if ctx == nil { ctx = context.Background() } start := time.Now() - byDeployment := make(map[string][]config.DecisionSignalRule) - stateOf := make(map[string]string) - var order []string + byCall := make(map[decisionCall][]config.DecisionSignalRule) + stateOf := make(map[decisionCall]string) + var order []decisionCall for _, rule := range c.Config.DecisionRules { if !signalRuleUsed(usedSignals, config.SignalTypeDecision, rule.Name) { continue } - deployment := c.Config.DecisionQuestionDeployment(rule) - if _, seen := byDeployment[deployment]; !seen { - order = append(order, deployment) - stateOf[deployment] = wholeText + call := decisionCall{c.Config.DecisionQuestionDeployment(rule), min(rule.PriorUserTurns, len(priorUserMessages))} + if _, seen := byCall[call]; !seen { + order = append(order, call) + stateOf[call] = withPriorUserTurns(wholeText, priorUserMessages, call.priorTurns) } - byDeployment[deployment] = append(byDeployment[deployment], rule) + byCall[call] = append(byCall[call], rule) } modelservice.Fan(ctx, len(order), func(i int) { - c.evaluateDecisionDeployment(ctx, results, mu, stateOf[order[i]], order[i], byDeployment[order[i]]) + c.evaluateDecisionDeployment(ctx, results, mu, stateOf[order[i]], order[i].deployment, byCall[order[i]]) }) mu.Lock() results.Metrics.Decision.ExecutionTimeMs = float64(time.Since(start).Microseconds()) / 1000.0 mu.Unlock() } +// withPriorUserTurns is state after the last turns of prior, oldest first, +// one blank line apart. The earlier turns share semanticSignalUnitLimit runes, +// the latest turn first and each cut to its first runes, so however many turns +// a question reads they add the same bounded context to the current turn. +func withPriorUserTurns(state string, prior []string, turns int) string { + parts := []string{state} + room := semanticSignalUnitLimit + for index := len(prior) - 1; index >= len(prior)-turns && room > 0; index-- { + runes := []rune(prior[index]) + runes = runes[:min(len(runes), room)] + room -= len(runes) + parts = append([]string{string(runes)}, parts...) + } + return strings.Join(parts, "\n\n") +} + func (c *Classifier) evaluateDecisionDeployment( ctx context.Context, results *SignalResults, diff --git a/src/semantic-router/pkg/classification/classifier_signal_decision_model_test.go b/src/semantic-router/pkg/classification/classifier_signal_decision_model_test.go index 53abf0e080f..d9c6e27b8ab 100644 --- a/src/semantic-router/pkg/classification/classifier_signal_decision_model_test.go +++ b/src/semantic-router/pkg/classification/classifier_signal_decision_model_test.go @@ -3,6 +3,7 @@ package classification import ( "context" "slices" + "strings" "sync" "testing" @@ -84,7 +85,7 @@ func TestDecisionSignalsBatchPerDeploymentAndMatch(t *testing.T) { for _, rule := range decisionRulesForTest() { used["decision:"+rule.Name] = true } - classifier.evaluateDecisionModelSignals(context.Background(), results, &sync.Mutex{}, "merge two lists", "merge two lists", used) + classifier.evaluateDecisionModelSignals(context.Background(), results, &sync.Mutex{}, "merge two lists", "merge two lists", nil, used) if decider.calls["kai"] != 1 || decider.calls["vega"] != 1 { t.Fatalf("one call per deployment expected, got %v", decider.calls) @@ -169,7 +170,7 @@ func TestSetAndSpanAnswersMatchLabelsAndPublishEveryProbability(t *testing.T) { func TestUnusedDecisionSignalsAreNotAsked(t *testing.T) { decider := &fakeDecider{calls: map[string]int{}, requests: map[string]modelservice.Request{}, answers: map[string]modelservice.Answer{}} classifier := newDecisionTestClassifier(decider) - classifier.evaluateDecisionModelSignals(context.Background(), newSignalResults(), &sync.Mutex{}, "x", "x", map[string]bool{"decision:remote": true}) + classifier.evaluateDecisionModelSignals(context.Background(), newSignalResults(), &sync.Mutex{}, "x", "x", nil, map[string]bool{"decision:remote": true}) if decider.calls["kai"] != 0 || decider.calls["vega"] != 1 { t.Fatalf("only used signals are asked: %v", decider.calls) } @@ -201,7 +202,7 @@ func TestADecisionQuestionWithoutDeploymentJoinsTheDecisionModelsCall(t *testing classifier := &Classifier{Config: cfg} classifier.SetDecisionDecider(decider) used := map[string]bool{"decision:tools": true, "decision:hard": true} - classifier.evaluateDecisionModelSignals(context.Background(), newSignalResults(), &sync.Mutex{}, "short", "the whole text", used) + classifier.evaluateDecisionModelSignals(context.Background(), newSignalResults(), &sync.Mutex{}, "short", "the whole text", nil, used) if decider.states[config.DefaultDecisionDeployment] != "the whole text" || decider.states["kai"] != "the whole text" { t.Fatalf("every decision question reads the same authored input: %v", decider.states) @@ -210,3 +211,74 @@ func TestADecisionQuestionWithoutDeploymentJoinsTheDecisionModelsCall(t *testing t.Fatalf("only a declared deployment's call takes the default timeout: %v", decider.deadlines) } } + +type recordingDecider struct { + mu sync.Mutex + requests []modelservice.Request +} + +func (r *recordingDecider) Decide(_ context.Context, _ string, request modelservice.Request) (modelservice.Response, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.requests = append(r.requests, request) + return modelservice.Response{Answers: map[string]modelservice.Answer{}}, nil +} + +// A follow-up such as "now make it shorter" says nothing about its task; +// a question with prior_user_turns reads it after the turns it follows, +// and only questions that read as many turns share its call. +func TestDecisionQuestionsReadTheirPriorUserTurns(t *testing.T) { + decider := &recordingDecider{} + cfg := &config.RouterConfig{} + cfg.DecisionRules = []config.DecisionSignalRule{ + {Name: "alone", Deployment: "kai", Question: config.DecisionQuestion{Type: "noul", Instructions: "?"}}, + {Name: "one", Deployment: "kai", PriorUserTurns: 1, Question: config.DecisionQuestion{Type: "noul", Instructions: "?"}}, + {Name: "many", Deployment: "kai", PriorUserTurns: 5, Question: config.DecisionQuestion{Type: "noul", Instructions: "?"}}, + } + classifier := &Classifier{Config: cfg} + classifier.SetDecisionDecider(decider) + used := map[string]bool{"decision:alone": true, "decision:one": true, "decision:many": true} + latest := strings.Repeat("ü", semanticSignalUnitLimit-10) + prior := []string{"Write a poem about rain.", latest} + + classifier.evaluateDecisionModelSignals(context.Background(), newSignalResults(), &sync.Mutex{}, "Now make it shorter.", "Now make it shorter.", prior, used) + + states := map[string]string{} + for _, request := range decider.requests { + for _, question := range request.Questions { + states[question.ID] = request.State + } + } + want := map[string]string{ + "alone": "Now make it shorter.", + "one": latest + "\n\nNow make it shorter.", + // five asked, two exist: oldest first, within one shared budget + "many": "Write a po\n\n" + latest + "\n\nNow make it shorter.", + } + for name, state := range want { + if states[name] != state { + t.Errorf("%s: state %q, want %q", name, states[name], state) + } + } + if len(decider.requests) != 3 { + t.Fatalf("questions reading different turns need separate calls, got %d", len(decider.requests)) + } +} + +func TestPriorUserTurnsWithoutHistoryShareTheCurrentTurnsCall(t *testing.T) { + decider := &recordingDecider{} + cfg := &config.RouterConfig{} + cfg.DecisionRules = []config.DecisionSignalRule{ + {Name: "alone", Deployment: "kai", Question: config.DecisionQuestion{Type: "noul", Instructions: "?"}}, + {Name: "one", Deployment: "kai", PriorUserTurns: 1, Question: config.DecisionQuestion{Type: "noul", Instructions: "?"}}, + } + classifier := &Classifier{Config: cfg} + classifier.SetDecisionDecider(decider) + used := map[string]bool{"decision:alone": true, "decision:one": true} + + classifier.evaluateDecisionModelSignals(context.Background(), newSignalResults(), &sync.Mutex{}, "hello", "hello", nil, used) + + if len(decider.requests) != 1 || decider.requests[0].State != "hello" || len(decider.requests[0].Questions) != 2 { + t.Fatalf("a first turn has no earlier turns, so both questions share one call: %+v", decider.requests) + } +} diff --git a/src/semantic-router/pkg/classification/classifier_signal_dispatch_policy.go b/src/semantic-router/pkg/classification/classifier_signal_dispatch_policy.go index 97ac19c5b37..9164b6edc79 100644 --- a/src/semantic-router/pkg/classification/classifier_signal_dispatch_policy.go +++ b/src/semantic-router/pkg/classification/classifier_signal_dispatch_policy.go @@ -106,6 +106,7 @@ func (c *Classifier) buildPolicySignalDispatchers( mu, textForSignal(config.SignalTypeDecision), textForSignal(decisionModelQuestionText), + priorUserMessages, usedSignals, ) }, diff --git a/src/semantic-router/pkg/config/decision_signal.go b/src/semantic-router/pkg/config/decision_signal.go index 313d1f5d5db..184c222e50b 100644 --- a/src/semantic-router/pkg/config/decision_signal.go +++ b/src/semantic-router/pkg/config/decision_signal.go @@ -29,12 +29,15 @@ const ( // DefaultDecisionTimeoutMs bounds one decision call when a rule sets none. DefaultDecisionTimeoutMs = 1000 MaxDecisionTimeoutMs = 60000 - MinDecisionChoices = 2 - MaxDecisionChoices = 255 - MinDecisionLevels = 2 - MaxDecisionLevels = 10 - MinDecisionLabels = 1 - MaxDecisionLabels = 255 + // MaxDecisionPriorUserTurns bounds how many earlier user turns a decision + // question reads before the current one. + MaxDecisionPriorUserTurns = 8 + MinDecisionChoices = 2 + MaxDecisionChoices = 255 + MinDecisionLevels = 2 + MaxDecisionLevels = 10 + MinDecisionLabels = 1 + MaxDecisionLabels = 255 // DefaultDecisionNoulThreshold is the P(true) a Noul answer needs to match // when its rule declares no predicate. DefaultDecisionNoulThreshold = 0.5 @@ -58,6 +61,11 @@ type DecisionSignalRule struct { Question DecisionQuestion `yaml:"question"` Predicate *NumericPredicate `yaml:"predicate,omitempty"` TimeoutMs int `yaml:"timeout_ms,omitempty"` + // PriorUserTurns puts up to this many of the conversation's earlier user + // turns, oldest first, before the current one in the question's state, + // so a follow-up such as "now make it shorter" is read with what it + // follows. + PriorUserTurns int `yaml:"prior_user_turns,omitempty"` } // DecisionQuestion is a System One question with ordered options. Choice and diff --git a/src/semantic-router/pkg/config/validator_decision_signal.go b/src/semantic-router/pkg/config/validator_decision_signal.go index 1fb5b086860..cab119c36f5 100644 --- a/src/semantic-router/pkg/config/validator_decision_signal.go +++ b/src/semantic-router/pkg/config/validator_decision_signal.go @@ -46,6 +46,9 @@ func ValidateDecisionSignalRuleContract(rule DecisionSignalRule) error { if rule.TimeoutMs < 0 || rule.TimeoutMs > MaxDecisionTimeoutMs { return fmt.Errorf("timeout_ms must be within [1, %d] when set", MaxDecisionTimeoutMs) } + if rule.PriorUserTurns < 0 || rule.PriorUserTurns > MaxDecisionPriorUserTurns { + return fmt.Errorf("prior_user_turns must be within [0, %d]", MaxDecisionPriorUserTurns) + } if err := validateDecisionQuestion(rule.Question); err != nil { return fmt.Errorf("question: %w", err) } diff --git a/src/semantic-router/pkg/config/validator_decision_signal_test.go b/src/semantic-router/pkg/config/validator_decision_signal_test.go index 82225b609a7..fe6babc0c75 100644 --- a/src/semantic-router/pkg/config/validator_decision_signal_test.go +++ b/src/semantic-router/pkg/config/validator_decision_signal_test.go @@ -90,6 +90,7 @@ func TestDecisionSignalContracts(t *testing.T) { "type": func(cfg *RouterConfig) { cfg.DecisionRules[0].Question.Type = "rank" }, "instructions": func(cfg *RouterConfig) { cfg.DecisionRules[0].Question.Instructions = " " }, "timeout": func(cfg *RouterConfig) { cfg.DecisionRules[0].TimeoutMs = MaxDecisionTimeoutMs + 1 }, + "prior turns": func(cfg *RouterConfig) { cfg.DecisionRules[0].PriorUserTurns = MaxDecisionPriorUserTurns + 1 }, "colon in name": func(cfg *RouterConfig) { cfg.DecisionRules[0].Name = "a:b" }, "input budget": func(cfg *RouterConfig) { decider := cfg.ModelDeployments["decider"] diff --git a/src/semantic-router/pkg/configschema/router-config-v0.3.schema.json b/src/semantic-router/pkg/configschema/router-config-v0.3.schema.json index 636fc44f22c..e1b878da677 100644 --- a/src/semantic-router/pkg/configschema/router-config-v0.3.schema.json +++ b/src/semantic-router/pkg/configschema/router-config-v0.3.schema.json @@ -2668,6 +2668,9 @@ }, "timeout_ms": { "type": "integer" + }, + "prior_user_turns": { + "type": "integer" } }, "additionalProperties": false, diff --git a/src/semantic-router/pkg/dsl/decision_model_roundtrip_test.go b/src/semantic-router/pkg/dsl/decision_model_roundtrip_test.go index ba59fd2a610..a1e84c250d6 100644 --- a/src/semantic-router/pkg/dsl/decision_model_roundtrip_test.go +++ b/src/semantic-router/pkg/dsl/decision_model_roundtrip_test.go @@ -18,7 +18,7 @@ func TestDecisionModelSignalAndAlgorithmRoundTrip(t *testing.T) { Predicate: &config.NumericPredicate{GTE: &threshold}, TimeoutMs: 800, }, { - Name: "request_kind", Deployment: "decision-kai", + Name: "request_kind", Deployment: "decision-kai", PriorUserTurns: 2, Question: config.DecisionQuestion{Type: "choice", Instructions: "Which kind?", Choices: []config.DecisionChoice{ {Key: "code", Description: "Code"}, {Key: "math"}, }}, diff --git a/src/semantic-router/pkg/dsl/decompiler_signals_rules.go b/src/semantic-router/pkg/dsl/decompiler_signals_rules.go index d0ba79f2aea..d0aee283b19 100644 --- a/src/semantic-router/pkg/dsl/decompiler_signals_rules.go +++ b/src/semantic-router/pkg/dsl/decompiler_signals_rules.go @@ -302,6 +302,9 @@ func (d *decompiler) decompileDecisionModelSignals() { if rule.TimeoutMs > 0 { d.write(" timeout_ms: %d\n", rule.TimeoutMs) } + if rule.PriorUserTurns > 0 { + d.write(" prior_user_turns: %d\n", rule.PriorUserTurns) + } d.write("}\n\n") } } diff --git a/src/vllm-sr/cli/models_decision.py b/src/vllm-sr/cli/models_decision.py index b5d89ff660d..d1121d3eb11 100644 --- a/src/vllm-sr/cli/models_decision.py +++ b/src/vllm-sr/cli/models_decision.py @@ -9,6 +9,7 @@ from .models_predicates import NumericPredicate MAX_DECISION_TIMEOUT_MS = 60_000 +MAX_DECISION_PRIOR_USER_TURNS = 8 MIN_DECISION_CHOICES = 2 MAX_DECISION_CHOICES = 255 MIN_DECISION_LEVELS = 2 @@ -117,6 +118,10 @@ class DecisionSignalRule(BaseModel): question: DecisionQuestion predicate: NumericPredicate | None = None timeout_ms: int | None = Field(default=None, ge=0, le=MAX_DECISION_TIMEOUT_MS) + # Earlier user turns read before the current one, oldest first. + prior_user_turns: int | None = Field( + default=None, ge=0, le=MAX_DECISION_PRIOR_USER_TURNS + ) @model_validator(mode="after") def validate_rule(self): diff --git a/src/vllm-sr/tests/test_cli_models.py b/src/vllm-sr/tests/test_cli_models.py index ef04b9098b3..40658ec5281 100644 --- a/src/vllm-sr/tests/test_cli_models.py +++ b/src/vllm-sr/tests/test_cli_models.py @@ -205,3 +205,28 @@ def test_custom_evaluation_requires_versioned_namespace(): }, } ) + + +def _decision_signal(**overrides): + rule = { + "name": "request_kind", + "deployment": "decision-kai", + "question": { + "type": "choice", + "instructions": "Which kind?", + "choices": [{"key": "code"}, {"key": "chat"}], + }, + } + rule.update(overrides) + return rule + + +def test_decision_signal_reads_up_to_eight_prior_user_turns(): + signals = {"decision": [_decision_signal(prior_user_turns=8)]} + config = UserConfig(version="0.3", routing={"signals": signals}) + assert config.signals.decision[0].prior_user_turns == 8 + with pytest.raises(ValueError): + UserConfig( + version="0.3", + routing={"signals": {"decision": [_decision_signal(prior_user_turns=9)]}}, + ) diff --git a/tools/agent/domains.yaml b/tools/agent/domains.yaml index 41d8b3922ef..7587b4a1c49 100644 --- a/tools/agent/domains.yaml +++ b/tools/agent/domains.yaml @@ -896,6 +896,7 @@ profiles: - e2e/pkg/modelruntime/** - e2e/testcases/model_runtime_*.go - e2e/testcases/decision_runtime_routing.go + - e2e/testcases/decision_prior_user_turns.go - src/model-runtime/Dockerfile - src/model-runtime/vllm_srun/** - src/semantic-router/pkg/modelservice/** diff --git a/website/docs/tutorials/signal/learned/decision.md b/website/docs/tutorials/signal/learned/decision.md index 05d631fce66..af4340d7247 100644 --- a/website/docs/tutorials/signal/learned/decision.md +++ b/website/docs/tutorials/signal/learned/decision.md @@ -186,6 +186,48 @@ routing: - model: large-reasoner ``` +### Follow-up turns + +A question reads the current user turn. A follow-up such as "Rewrite your +previous answer in fewer words" says little about its task on its own, so a +question can also read earlier user turns with `prior_user_turns` (0 to 8): + +```yaml alternative +global: + model_catalog: + deployments: + decision-kai: + provider: model_runtime + artifact: vllm-sr/Decision-2.0-Kai-0.6B + device: auto + +routing: + signals: + decision: + - name: request_kind + deployment: decision-kai + prior_user_turns: 1 + question: + type: choice + instructions: What kind of request is the latest user turn? + choices: + - key: code + description: Writing, reviewing or debugging code + - key: chat + description: Anything else +``` + +The state is then up to that many earlier user turns, oldest first, then the +current turn, one blank line apart. The earlier turns share 1,760 characters, +filled from the latest turn back, so they add the same bounded context however +many turns a question reads. Assistant and tool messages are not included. +Questions to one deployment share a call only when they read the same number of +turns. On +MT-Bench's 80 two-turn conversations, an eight-way Choice over its categories +on the second turn was right 41% to 55% of the time from that turn alone and +71% to 84% with the first turn read too (Decision 2.0 Kai-0.6B to Lux-9B), and +35% to 51% against 67.5% to 87.5% on Vela 2.0 0.3B to 9B. + ### Set and span questions A `set` question names labels and asks which apply; a `span` question asks diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/signal/learned/decision.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/signal/learned/decision.md index 2df2f8f2f74..f11f7b76eb1 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/signal/learned/decision.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/tutorials/signal/learned/decision.md @@ -167,6 +167,37 @@ routing: - model: large-reasoner ``` +### 后续轮次 + +问题读取当前用户轮次。像"Rewrite your previous answer in fewer words"这样的后续请求本身几乎说明不了任务,因此问题还可以通过 `prior_user_turns`(0 到 8)读取更早的用户轮次: + +```yaml alternative +global: + model_catalog: + deployments: + decision-kai: + provider: model_runtime + artifact: vllm-sr/Decision-2.0-Kai-0.6B + device: auto + +routing: + signals: + decision: + - name: request_kind + deployment: decision-kai + prior_user_turns: 1 + question: + type: choice + instructions: What kind of request is the latest user turn? + choices: + - key: code + description: Writing, reviewing or debugging code + - key: chat + description: Anything else +``` + +此时状态依次为最多这么多个更早的用户轮次(从最早开始),然后是当前轮次,中间空一行。更早的轮次共享 1,760 个字符,从最近的轮次往前填充,因此无论问题读取多少轮次,增加的上下文都有同样的上限。助手和工具消息不包含在内。发往同一部署的问题只有在读取相同轮次数时才共用一次调用。在 MT-Bench 的 80 段两轮对话上,对第二轮按其类别做八选一 Choice,仅看该轮时正确率为 41% 到 55%,同时读取第一轮时为 71% 到 84%(Decision 2.0 Kai-0.6B 到 Lux-9B);在 Vela 2.0 0.3B 到 9B 上分别为 35% 到 51% 和 67.5% 到 87.5%。 + ### Set 和 Span 问题 {#set-and-span-questions} `set` 问题列出标签并询问哪些适用;`span` 问题询问请求中每个标签出现的位置。两者都使用 `labels` 而非 `choices`,每个标签包含 `key` 和可选的 `description`;规则条件通过标签名引用它们: