diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 7d8b2fb66..8b146e039 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -597,6 +597,7 @@ struct ClassifierRouteConfig { } /// Complete construction settings for one LLM classifier mode. +#[derive(Clone)] #[non_exhaustive] pub enum LlmClassifierConfig { /// Routes between efficient and capable targets from a task-level verdict. diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index f52e29e36..5d6e018c1 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -10,8 +10,10 @@ use futures::StreamExt; use http::header::{HeaderName, HeaderValue}; use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError}; use pyo3::prelude::*; +use serde_json::Value; use switchyard_libsy::{ - Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, HandoffNoteConfig, + Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, + CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, StageRouter, StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, @@ -57,6 +59,190 @@ impl PyTaskClassifierConfig { } } +/// Settings for response-based escalation classification. +#[pyclass( + name = "EscalationClassifierConfig", + module = "switchyard.libsy", + frozen, + skip_from_py_object +)] +#[derive(Clone)] +struct PyEscalationClassifierConfig { + contract: ClassifierContractConfig, + judge: EscalationJudgeConfig, + max_output_tokens: u64, +} + +#[pymethods] +impl PyEscalationClassifierConfig { + #[new] + #[pyo3(signature = ( + *, + confirmations=2, + recent_turn_window=28, + window_message_chars=500, + max_output_tokens=4096, + prompt=None, + response_format_type="json_schema" + ))] + #[allow(clippy::too_many_arguments)] + fn new( + confirmations: u32, + recent_turn_window: usize, + window_message_chars: usize, + max_output_tokens: u64, + prompt: Option, + response_format_type: &str, + ) -> PyResult { + Ok(Self { + contract: classifier_contract(prompt, response_format_type)?, + judge: EscalationJudgeConfig { + confirmations, + recent_turn_window, + window_message_chars, + }, + max_output_tokens, + }) + } +} + +/// Settings for a classifier with a user-supplied verdict schema. +#[pyclass( + name = "CustomClassifierConfig", + module = "switchyard.libsy", + frozen, + skip_from_py_object +)] +#[derive(Clone)] +struct PyCustomClassifierConfig { + inner: CustomClassifierConfig, +} + +impl PyCustomClassifierConfig { + fn clone_core(&self) -> CustomClassifierConfig { + self.inner.clone() + } +} + +#[pymethods] +impl PyCustomClassifierConfig { + #[new] + #[pyo3(signature = ( + prompt, + response_schema, + selector, + *, + session_affinity=false, + message_hash_fallback=false, + recent_turn_window=None, + max_output_tokens=4096 + ))] + #[allow(clippy::too_many_arguments)] + fn new( + prompt: String, + response_schema: &Bound<'_, PyAny>, + selector: String, + session_affinity: bool, + message_hash_fallback: bool, + recent_turn_window: Option, + max_output_tokens: u64, + ) -> PyResult { + // Convert the Python schema into serde JSON and pair it with the target-selector policy; + // conversion failures propagate to Python through `PyResult`. + let mut inner = CustomClassifierConfig::new( + prompt, + from_python::(response_schema)?, + CustomClassifierPolicy::target_selector(selector), + ); + inner.session_affinity = session_affinity; + inner.message_hash_fallback = message_hash_fallback; + inner.recent_turn_window = recent_turn_window; + inner.max_output_tokens = max_output_tokens; + Ok(Self { inner }) + } +} + +/// Construction settings for a Python-hosted LLM classifier. +#[pyclass( + name = "LlmClassifierConfig", + module = "switchyard.libsy", + frozen, + skip_from_py_object +)] +struct PyLlmClassifierConfig { + inner: LlmClassifierConfig, +} + +#[pymethods] +impl PyLlmClassifierConfig { + /// Configure capability routing between efficient and capable targets. + #[staticmethod] + #[pyo3(signature = (judge_target, efficient_target, capable_target, *, config))] + fn capability( + py: Python<'_>, + judge_target: String, + efficient_target: String, + capable_target: String, + config: Py, + ) -> PyResult { + Ok(Self { + inner: LlmClassifierConfig::Capability { + judge_target: ModelId::new(judge_target), + efficient_target: ModelId::new(efficient_target), + capable_target: ModelId::new(capable_target), + config: config.bind(py).try_borrow()?.clone_core(), + }, + }) + } + + /// Configure response-based escalation between efficient and capable targets. + #[staticmethod] + #[pyo3(signature = (judge_target, efficient_target, capable_target, *, config))] + fn escalation( + py: Python<'_>, + judge_target: String, + efficient_target: String, + capable_target: String, + config: Py, + ) -> PyResult { + let config = config.bind(py).try_borrow()?; + Ok(Self { + inner: LlmClassifierConfig::Escalation { + judge_target: ModelId::new(judge_target), + efficient_target: ModelId::new(efficient_target), + capable_target: ModelId::new(capable_target), + contract: config.contract.clone(), + config: config.judge.clone(), + max_output_tokens: config.max_output_tokens, + }, + }) + } + + /// Configure schema-driven routing across named targets. + #[staticmethod] + #[pyo3(signature = (judge_target, targets, *, default_target, config))] + fn custom( + py: Python<'_>, + judge_target: String, + targets: Vec<(String, String)>, + default_target: String, + config: Py, + ) -> PyResult { + let config = config.bind(py).try_borrow()?.clone_core(); + Ok(Self { + inner: LlmClassifierConfig::Custom { + judge_target: ModelId::new(judge_target), + targets: targets + .into_iter() + .map(|(name, target)| (name, ModelId::new(target))) + .collect(), + default_target, + config, + }, + }) + } +} + #[pymethods] impl PyTaskClassifierConfig { #[new] @@ -82,20 +268,6 @@ impl PyTaskClassifierConfig { prompt: Option, response_format_type: &str, ) -> PyResult { - let mut contract = ClassifierContractConfig::default(); - if let Some(prompt) = prompt { - contract = contract.with_prompt(prompt); - } - let response_format_type = match response_format_type { - "json_schema" => ClassifierResponseFormat::JsonSchema, - "json_object" => ClassifierResponseFormat::JsonObject, - other => { - return Err(PyValueError::new_err(format!( - "response_format_type must be 'json_schema' or 'json_object', got {other:?}" - ))); - } - }; - contract = contract.with_response_format_type(response_format_type); Ok(Self { inner: TaskClassifierConfig { base_threshold, @@ -103,13 +275,33 @@ impl PyTaskClassifierConfig { session_affinity, message_hash_fallback, recent_turn_window, - contract, + contract: classifier_contract(prompt, response_format_type)?, max_output_tokens, }, }) } } +fn classifier_contract( + prompt: Option, + response_format_type: &str, +) -> PyResult { + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = prompt { + contract = contract.with_prompt(prompt); + } + let response_format_type = match response_format_type { + "json_schema" => ClassifierResponseFormat::JsonSchema, + "json_object" => ClassifierResponseFormat::JsonObject, + other => { + return Err(PyValueError::new_err(format!( + "response_format_type must be 'json_schema' or 'json_object', got {other:?}" + ))); + } + }; + Ok(contract.with_response_format_type(response_format_type)) +} + /// Judge target and policy used when stage-router signals are inconclusive. #[pyclass( name = "LlmFallback", @@ -419,7 +611,16 @@ fn random_algorithm( }) } -/// Construct task-level LLM classifier routing. +/// Construct LLM classifier routing from a mode config. +#[pyfunction(name = "llm_classifier")] +fn llm_classifier_algorithm( + py: Python<'_>, + config: Py, +) -> PyResult { + build_llm_classifier(config.bind(py).try_borrow()?.inner.clone()) +} + +/// Construct capability classifier routing. #[pyfunction(name = "llm_task_classifier")] #[pyo3(signature = ( judge_target, @@ -435,13 +636,17 @@ fn llm_task_classifier_algorithm( capable_target: String, config: Py, ) -> PyResult { - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability { + build_llm_classifier(LlmClassifierConfig::Capability { judge_target: ModelId::new(judge_target), efficient_target: ModelId::new(efficient_target), capable_target: ModelId::new(capable_target), config: config.bind(py).try_borrow()?.clone_core(), }) - .map_err(|error| PyValueError::new_err(error.to_string()))?; +} + +fn build_llm_classifier(config: LlmClassifierConfig) -> PyResult { + let algorithm = + LlmTaskClassifier::new(config).map_err(|error| PyValueError::new_err(error.to_string()))?; Ok(PyAlgorithm { inner: Arc::new(algorithm), }) @@ -524,7 +729,10 @@ fn stage_router_algorithm( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let libsy_module = PyModule::new(module.py(), "libsy")?; libsy_module.add_class::()?; + libsy_module.add_class::()?; libsy_module.add_class::()?; + libsy_module.add_class::()?; + libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; libsy_module.add_class::()?; @@ -532,6 +740,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { libsy_module.add_class::()?; libsy_module.add_function(wrap_pyfunction!(noop_algorithm, &libsy_module)?)?; libsy_module.add_function(wrap_pyfunction!(random_algorithm, &libsy_module)?)?; + libsy_module.add_function(wrap_pyfunction!(llm_classifier_algorithm, &libsy_module)?)?; libsy_module.add_function(wrap_pyfunction!( llm_task_classifier_algorithm, &libsy_module diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py index 78a535d01..69e5f26c5 100644 --- a/switchyard/libsy/__init__.py +++ b/switchyard/libsy/__init__.py @@ -6,8 +6,11 @@ from switchyard_rust.libsy import ( Algorithm, ContextWindowExceededError, + CustomClassifierConfig, Decision, + EscalationClassifierConfig, LibsyError, + LlmClassifierConfig, LlmFallback, ModelCall, Step, @@ -19,8 +22,11 @@ __all__ = [ "Algorithm", "ContextWindowExceededError", + "CustomClassifierConfig", "Decision", + "EscalationClassifierConfig", "LibsyError", + "LlmClassifierConfig", "LlmFallback", "ModelCall", "Step", diff --git a/switchyard/libsy/algorithms.py b/switchyard/libsy/algorithms.py index 92e9e747e..444f9598c 100644 --- a/switchyard/libsy/algorithms.py +++ b/switchyard/libsy/algorithms.py @@ -3,9 +3,10 @@ """Factories for Rust-owned libsy algorithms.""" +from switchyard_rust.libsy import llm_classifier as llm_classifier from switchyard_rust.libsy import llm_task_classifier as llm_task_classifier from switchyard_rust.libsy import noop as noop from switchyard_rust.libsy import random as random from switchyard_rust.libsy import stage_router as stage_router -__all__ = ["llm_task_classifier", "noop", "random", "stage_router"] +__all__ = ["llm_classifier", "llm_task_classifier", "noop", "random", "stage_router"] diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 67e4e1888..65542dd68 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -14,12 +14,16 @@ { "Algorithm", "ContextWindowExceededError", + "CustomClassifierConfig", "Decision", + "EscalationClassifierConfig", "LibsyError", + "LlmClassifierConfig", "LlmFallback", "ModelCall", "Step", "TaskClassifierConfig", + "llm_classifier", "llm_task_classifier", "noop", "random", @@ -35,6 +39,45 @@ class LibsyError(RuntimeError): ... class ContextWindowExceededError(RuntimeError): ... + @final + class CustomClassifierConfig: + """Configure schema-validated routing across named targets. + + ``max_output_tokens`` must be positive. Enabling ``message_hash_fallback`` + requires ``session_affinity``. + """ + + def __init__( + self, + prompt: str, + response_schema: Mapping[str, object], + selector: str, + *, + session_affinity: bool = False, + message_hash_fallback: bool = False, + recent_turn_window: int | None = None, + max_output_tokens: int = 4096, + ) -> None: ... + + @final + class EscalationClassifierConfig: + """Configure response-based escalation between two targets. + + Counts and token limits must be positive, and ``window_message_chars`` + must be at least 50. + """ + + def __init__( + self, + *, + confirmations: int = 2, + recent_turn_window: int = 28, + window_message_chars: int = 500, + max_output_tokens: int = 4096, + prompt: str | None = None, + response_format_type: Literal["json_schema", "json_object"] = "json_schema", + ) -> None: ... + @final class Decision: """A semantic routing choice produced by an algorithm.""" @@ -88,6 +131,12 @@ class Done: @final class TaskClassifierConfig: + """Configure capability classification between efficient and capable targets. + + Thresholds must remain within ``[0, 1]``, ``max_output_tokens`` must be + positive, and ``message_hash_fallback`` requires ``session_affinity``. + """ + def __init__( self, base_threshold: float, @@ -101,6 +150,46 @@ def __init__( response_format_type: Literal["json_schema", "json_object"] = "json_schema", ) -> None: ... + class LlmClassifierConfig: + """Select one supported LLM classifier mode. + + Target names and each nested mode configuration must satisfy the selected + classifier's invariants. + """ + + @staticmethod + def capability( + judge_target: str, + efficient_target: str, + capable_target: str, + *, + config: TaskClassifierConfig, + ) -> LlmClassifierConfig: + """Route by predicted task capability.""" + ... + + @staticmethod + def escalation( + judge_target: str, + efficient_target: str, + capable_target: str, + *, + config: EscalationClassifierConfig, + ) -> LlmClassifierConfig: + """Call the efficient target first and escalate judged responses.""" + ... + + @staticmethod + def custom( + judge_target: str, + targets: Sequence[tuple[str, str]], + *, + default_target: str, + config: CustomClassifierConfig, + ) -> LlmClassifierConfig: + """Route among named targets using a schema-selected label.""" + ... + @final class LlmFallback: def __init__( @@ -127,6 +216,10 @@ def random( seed: int | None = None, ) -> Algorithm: ... + def llm_classifier(config: LlmClassifierConfig) -> Algorithm: + """Build a classifier, raising ValueError when its configuration is invalid.""" + ... + def llm_task_classifier( judge_target: str, efficient_target: str, diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 8e93cd6bb..a01d61d03 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -10,8 +10,10 @@ from switchyard.libsy import ( Algorithm, ContextWindowExceededError, + CustomClassifierConfig, Decision, LibsyError, + LlmClassifierConfig, Step, TaskClassifierConfig, algorithms, @@ -161,14 +163,16 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: judge = JudgeClient("judge") weak = EchoClient("weak") - algorithm = algorithms.llm_task_classifier( - "judge", - "weak", - "strong", - config=TaskClassifierConfig( - 0.5, - threshold_step=0.1, - prompt="Custom capability rubric.", + algorithm = algorithms.llm_classifier( + LlmClassifierConfig.capability( + "judge", + "weak", + "strong", + config=TaskClassifierConfig( + 0.5, + threshold_step=0.1, + prompt="Custom capability rubric.", + ), ), ) @@ -189,6 +193,49 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "weak" +async def test_custom_classifier_routes_across_named_targets() -> None: + class JudgeClient(EchoClient): + async def call(self, request: dict[str, Any]) -> dict[str, Any]: + self.calls.append(request) + return { + "model": self.model, + "outputs": [ + { + "role": "assistant", + "content": [{"type": "text", "text": '{"target":"balanced"}'}], + "stop_reason": "end_turn", + } + ], + } + + schema = { + "type": "object", + "additionalProperties": False, + "required": ["target"], + "properties": {"target": {"type": "string", "enum": ["fast", "balanced", "best"]}}, + } + algorithm = algorithms.llm_classifier( + LlmClassifierConfig.custom( + "judge", + [("fast", "model-a"), ("balanced", "model-b"), ("best", "model-c")], + default_target="fast", + config=CustomClassifierConfig("Choose a target.", schema, "/target"), + ) + ) + + _, response = await run_algorithm( + algorithm, + { + "judge": JudgeClient("judge"), + "model-a": EchoClient("model-a"), + "model-b": EchoClient("model-b"), + "model-c": EchoClient("model-c"), + }, + ) + + assert response["model"] == "model-b" + + async def test_classifier_config_accepts_json_object_output() -> None: """Verify that Python can select JSON Object mode for a classifier judge."""