feat(proxy, admin): runtime model status, cooldown, background health check, /admin/v1/models/status

crates/aisix-admin/src/lib.rs

@@ -39,6 +39,7 @@ pub mod etcd_store;
 mod guardrails_handlers;
 mod health_handler;
 mod models_handlers;
+mod models_status_handler;
 mod observability_exporters_handlers;
 mod openapi;
 mod playground_handler;
@@ -73,6 +74,10 @@ pub fn build_router(state: AdminState) -> Router {
                 .put(models_handlers::update_model)
                 .delete(models_handlers::delete_model),
         )
+        .route(
+            "/admin/v1/models/status",
+            get(models_status_handler::get_models_status),
+        )
         .route(
             "/admin/v1/apikeys",
             get(apikeys_handlers::list_apikeys).post(apikeys_handlers::create_apikey),
@@ -1089,4 +1094,92 @@ mod tests {
         // Empty snapshot → empty model list, but endpoint is operational.
         assert!(v["models"].is_array());
     }
+
+    #[tokio::test]
+    async fn models_status_returns_direct_and_routing_rows() {
+        use aisix_core::resource::ResourceEntry;
+        use aisix_core::Model;
+        use aisix_proxy::ModelRuntimeStatusTracker;
+
+        let handle = SnapshotHandle::new(AisixSnapshot::new());
+        let store = InMemoryStore::new() as Arc<dyn ConfigStore>;
+        let runtime = Arc::new(ModelRuntimeStatusTracker::new());
+
+        let direct: Model = serde_json::from_value(model_payload("gpt4")).unwrap();
+        store
+            .put_model(ResourceEntry {
+                id: "direct-1".into(),
+                value: direct,
+                revision: 1,
+            })
+            .await
+            .unwrap();
+
+        let routing: Model = serde_json::from_value(json!({
+            "display_name": "router",
+            "routing": {
+                "targets": [{"model": "gpt4"}]
+            }
+        }))
+        .unwrap();
+        store
+            .put_model(ResourceEntry {
+                id: "routing-1".into(),
+                value: routing,
+                revision: 1,
+            })
+            .await
+            .unwrap();
+
+        runtime.record_ignored_check("direct-1", 429, "ignored_transient_error");
+
+        let state = AdminState::new(handle, store, &cfg()).with_runtime_status_tracker(runtime);
+        let app = build_router(state);
+        let resp = run(app, auth_req("GET", "/admin/v1/models/status", None)).await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        let rows = body_json(resp).await;
+        let rows = rows.as_array().unwrap();
+        assert_eq!(rows.len(), 2);
+
+        let direct = rows.iter().find(|row| row["id"] == "direct-1").unwrap();
+        assert_eq!(direct["kind"], "direct");
+        assert_eq!(direct["status"], "healthy");
+        assert_eq!(direct["last_check_status"], 429);
+        assert_eq!(direct["status_reason"], "ignored_transient_error");
+
+        let routing = rows.iter().find(|row| row["id"] == "routing-1").unwrap();
+        assert_eq!(routing["kind"], "routing");
+        assert_eq!(routing["status"], "not_applicable");
+    }
+
+    #[tokio::test]
+    async fn create_model_accepts_background_model_check() {
+        let app = build_router(build_state());
+        let resp = run(
+            app,
+            auth_req(
+                "POST",
+                "/admin/v1/models",
+                Some(json!({
+                    "display_name": "bg-model",
+                    "provider": "openai",
+                    "model_name": "gpt-4o-mini",
+                    "provider_key_id": "11111111-1111-1111-1111-111111111111",
+                    "background_model_check": {
+                        "enabled": true,
+                        // Minimum interval is 5s in schema; using 30 to
+                        // mirror a realistic operator config.
+                        "interval_seconds": 30,
+                        "timeout_seconds": 10,
+                        "prompt": "Respond with OK",
+                        "max_tokens": 8,
+                        "ignore_statuses": [408, 429],
+                        "stale_after_seconds": 90
+                    }
+                })),
+            ),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+    }
 }

crates/aisix-admin/src/models_status_handler.rs

@@ -0,0 +1,72 @@
+//! `GET /admin/v1/models/status` — runtime per-model status.
+
+use axum::extract::State;
+use axum::Json;
+use serde::Serialize;
+use std::time::Duration;
+
+use aisix_proxy::{RuntimeStatus, RuntimeStatusSnapshot};
+
+use crate::auth::AdminAuth;
+use crate::error::AdminError;
+use crate::state::AdminState;
+
+#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum ModelKind {
+    Direct,
+    Routing,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ModelStatusView {
+    pub id: String,
+    pub display_name: String,
+    pub kind: ModelKind,
+    #[serde(flatten)]
+    pub details: RuntimeStatusSnapshot,
+}
+
+pub async fn get_models_status(
+    _auth: AdminAuth,
+    State(state): State<AdminState>,
+) -> Result<Json<Vec<ModelStatusView>>, AdminError> {
+    let all_models = state.store.list_models().await?;
+    let tracker = state.runtime_status_tracker.as_ref();
+
+    let models = all_models
+        .into_iter()
+        .map(|entry| {
+            if entry.value.is_routing() {
+                ModelStatusView {
+                    id: entry.id,
+                    display_name: entry.value.display_name,
+                    kind: ModelKind::Routing,
+                    details: RuntimeStatusSnapshot {
+                        status: RuntimeStatus::NotApplicable,
+                        ..RuntimeStatusSnapshot::default()
+                    },
+                }
+            } else {
+                let details = tracker
+                    .map(|t| {
+                        let stale_after = entry
+                            .value
+                            .background_model_check
+                            .as_ref()
+                            .map(|cfg| Duration::from_secs(cfg.stale_after_seconds));
+                        t.status_with_stale(&entry.id, stale_after)
+                    })
+                    .unwrap_or_default();
+                ModelStatusView {
+                    id: entry.id,
+                    display_name: entry.value.display_name,
+                    kind: ModelKind::Direct,
+                    details,
+                }
+            }
+        })
+        .collect();
+
+    Ok(Json(models))
+}

crates/aisix-admin/src/openapi.rs

@@ -115,6 +115,18 @@ const OPENAPI_JSON: &str = r##"{
       },
       "delete": { "summary": "delete model", "responses": {"200": {"description": "OK"}, "404": {"description": "not found"}} }
     },
+    "/admin/v1/models/status": {
+      "get": {
+        "summary": "list per-model runtime status",
+        "description": "Returns runtime routing/exclusion state for every Model. Direct models surface live runtime state keyed by resolved direct-model id; routing models return `not_applicable`. Request-path retryable failures surface as `cooldown`. Background checks can surface `unhealthy` or a healthy row with `status_reason=ignored_transient_error`.",
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {"application/json": {"schema": {"type": "array", "items": {"$ref": "#/components/schemas/ModelStatusView"}}}}
+          }
+        }
+      }
+    },
     "/admin/v1/apikeys": {
       "get":  { "summary": "list api keys",  "responses": {"200": {"description": "OK"}} },
       "post": {
@@ -265,9 +277,10 @@ const OPENAPI_JSON: &str = r##"{
           "timeout":         {"type": "integer", "minimum": 0, "description": "Request timeout in milliseconds. Absent or 0 = no timeout."},
           "rate_limit":      {"$ref": "#/components/schemas/RateLimit"},
           "routing":         {"$ref": "#/components/schemas/Routing"},
-          "cost":            {"$ref": "#/components/schemas/ModelCost"}
+          "cost":            {"$ref": "#/components/schemas/ModelCost"},
+          "background_model_check": {"$ref": "#/components/schemas/BackgroundModelCheck"}
         },
-        "description": "A direct model ships `provider` + `model_name` + `provider_key_id`; a routing model ships `routing` and omits the upstream triple."
+        "description": "A direct model ships `provider` + `model_name` + `provider_key_id`; a routing model ships `routing` and omits the upstream triple. `background_model_check` is direct-model-only and rejected on routing models."
       },
       "ModelEntry": {
         "type": "object",
@@ -278,6 +291,55 @@ const OPENAPI_JSON: &str = r##"{
           "revision": {"type": "integer"}
         }
       },
+      "BackgroundModelCheck": {
+        "type": "object",
+        "required": ["enabled", "interval_seconds", "timeout_seconds", "prompt", "max_tokens", "stale_after_seconds"],
+        "properties": {
+          "enabled": {"type": "boolean", "description": "Turns the periodic direct-model probe on or off."},
+          "interval_seconds": {"type": "integer", "minimum": 1, "description": "Probe interval in seconds."},
+          "timeout_seconds": {"type": "integer", "minimum": 1, "description": "Per-probe timeout in seconds."},
+          "prompt": {"type": "string", "minLength": 1, "description": "Minimal prompt used by the background probe request."},
+          "max_tokens": {"type": "integer", "minimum": 1, "description": "Max completion tokens used by the probe request."},
+          "ignore_statuses": {
+            "type": "array",
+            "description": "Upstream HTTP statuses that should be recorded without marking the model unhealthy. Typical values are 408 and 429.",
+            "items": {"type": "integer", "minimum": 100, "maximum": 599}
+          },
+          "stale_after_seconds": {"type": "integer", "minimum": 1, "description": "Age threshold after which an unhealthy background-check result is treated as stale and stops excluding the model."}
+        },
+        "description": "Periodic direct-model health-check configuration. Rejected on routing models."
+      },
+      "ModelStatusView": {
+        "type": "object",
+        "required": ["id", "display_name", "kind", "status"],
+        "properties": {
+          "id": {"type": "string", "description": "Resolved model id. Direct-model runtime status is keyed by this id."},
+          "display_name": {"type": "string"},
+          "kind": {"$ref": "#/components/schemas/ModelKind"},
+          "status": {"$ref": "#/components/schemas/RuntimeStatus"},
+          "cooldown_until": {"$ref": "#/components/schemas/SystemTime"},
+          "last_checked_at": {"$ref": "#/components/schemas/SystemTime"},
+          "last_check_status": {"type": "integer", "minimum": 100, "maximum": 599},
+          "status_reason": {"type": "string", "description": "Machine-readable explanation such as `retryable_failure`, `background_check_failed`, or `ignored_transient_error`."}
+        },
+        "description": "Per-model runtime routing status. Routing rows always return `kind=routing` and `status=not_applicable`."
+      },
+      "ModelKind": {
+        "type": "string",
+        "enum": ["direct", "routing"]
+      },
+      "RuntimeStatus": {
+        "type": "string",
+        "enum": ["healthy", "unhealthy", "cooldown", "not_applicable"]
+      },
+      "SystemTime": {
+        "type": "object",
+        "required": ["secs_since_epoch", "nanos_since_epoch"],
+        "properties": {
+          "secs_since_epoch": {"type": "integer", "minimum": 0},
+          "nanos_since_epoch": {"type": "integer", "minimum": 0, "maximum": 999999999}
+        }
+      },
       "ApiKey": {
         "type": "object",
         "required": ["key_hash", "allowed_models"],
@@ -436,6 +498,7 @@ mod tests {
             "/admin/openapi-scalar",
             "/admin/v1/models",
             "/admin/v1/models/{id}",
+            "/admin/v1/models/status",
             "/admin/v1/apikeys",
             "/admin/v1/apikeys/{id}",
             "/admin/v1/apikeys/{id}/rotate",
@@ -459,6 +522,11 @@ mod tests {
         for schema in [
             "Model",
             "ModelEntry",
+            "BackgroundModelCheck",
+            "ModelStatusView",
+            "ModelKind",
+            "RuntimeStatus",
+            "SystemTime",
             "ApiKey",
             "ApiKeyEntry",
             "ProviderKey",

crates/aisix-admin/src/state.rs

@@ -15,7 +15,7 @@ use aisix_core::snapshot::SnapshotHandle;
 use aisix_core::{AdminConfig, AisixSnapshot};
 use aisix_etcd::WatchStatus;
 use aisix_obs::Metrics;
-use aisix_proxy::{HealthTracker, LivezState};
+use aisix_proxy::{HealthTracker, LivezState, ModelRuntimeStatusTracker};
 use axum::Router;
 use std::sync::Arc;
 
@@ -30,6 +30,9 @@ pub struct AdminState {
     /// Shared in-process health tracker from the proxy. Used by the
     /// `/admin/v1/health` endpoint to report per-model health status.
     pub health_tracker: Option<Arc<HealthTracker>>,
+    /// Shared in-process runtime status tracker from the proxy. Used by
+    /// `/admin/v1/models/status` to report direct-model runtime state.
+    pub runtime_status_tracker: Option<Arc<ModelRuntimeStatusTracker>>,
     /// Watch supervisor's freshness state. When wired, the
     /// `/admin/v1/health` endpoint includes etcd revision +
     /// snapshot age so operators can detect a frozen / wedged config
@@ -57,6 +60,7 @@ impl AdminState {
             store,
             metrics: None,
             health_tracker: None,
+            runtime_status_tracker: None,
             watch_status: None,
             livez_state: Arc::new(LivezState::new()),
             proxy_router: None,
@@ -93,6 +97,12 @@ impl AdminState {
         self
     }
 
+    /// Attach the in-process runtime status tracker from the proxy.
+    pub fn with_runtime_status_tracker(mut self, tracker: Arc<ModelRuntimeStatusTracker>) -> Self {
+        self.runtime_status_tracker = Some(tracker);
+        self
+    }
+
     /// Wire the proxy router so the playground endpoint can forward
     /// requests to it in-process via `tower::ServiceExt::oneshot`.
     pub fn with_proxy_router(mut self, router: Router) -> Self {

crates/aisix-core/src/lib.rs

@@ -32,9 +32,10 @@ pub use error::{
 pub use models::{
     validate_apikey, validate_cache_policy, validate_guardrail, validate_model,
     validate_observability_exporter, validate_provider_key, AisixSnapshot, ApiKey, CachePolicy,
-    ExporterKind, Guardrail, GuardrailHookPoint, GuardrailKind, KeywordConfig, KeywordPattern,
-    Model, ObservabilityExporter, Provider, ProviderKey, RateLimit, Routing, RoutingStrategy,
-    RoutingTarget, SchemaError,
+    CooldownConfig, ExporterKind, Guardrail, GuardrailHookPoint, GuardrailKind, KeywordConfig,
+    KeywordPattern, Model, ObservabilityExporter, OnAllFilteredPolicy, Provider, ProviderKey,
+    RateLimit, Routing, RoutingStrategy, RoutingTarget, SchemaError,
+    DEFAULT_COOLDOWN_TRIGGER_STATUSES,
 };
 pub use resource::{Resource, ResourceEntry};
 pub use snapshot::{ResourceTable, SnapshotHandle};

crates/aisix-core/src/models/mod.rs

@@ -32,11 +32,13 @@ pub use guardrail::{
     BedrockAWSCredentials, BedrockConfig, BedrockLatencyMode, Guardrail, GuardrailHookPoint,
     GuardrailKind, KeywordConfig, KeywordPattern,
 };
-pub use model::{Model, Provider};
+pub use model::{
+    BackgroundModelCheck, CooldownConfig, Model, Provider, DEFAULT_COOLDOWN_TRIGGER_STATUSES,
+};
 pub use observability_exporter::{ExporterKind, ObservabilityExporter, OtlpHttpConfig};
 pub use provider_key::ProviderKey;
 pub use rate_limit::RateLimit;
-pub use routing::{Routing, RoutingStrategy, RoutingTarget};
+pub use routing::{OnAllFilteredPolicy, Routing, RoutingStrategy, RoutingTarget};
 pub use schema::{
     validate_apikey, validate_cache_policy, validate_guardrail, validate_model,
     validate_observability_exporter, validate_provider_key, SchemaError,