Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/app/api/endpoints/knowledge.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ def create_knowledge_base(
namespace=data.namespace or "default",
kb_type=data.kb_type or "notebook",
summary_enabled=data.summary_enabled,
rag_config_mode=data.rag_config_mode,
retrieval_config=(
data.retrieval_config.model_dump() if data.retrieval_config else None
),
Expand Down
2 changes: 1 addition & 1 deletion backend/app/api/endpoints/rag.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ async def retrieve_documents(
"namespace": "default"
},
"top_k": 5,
"score_threshold": 0.7,
"score_threshold": 0.5,
"retrieval_mode": "hybrid",
"hybrid_weights": {
"vector_weight": 0.7,
Expand Down
21 changes: 12 additions & 9 deletions backend/app/schemas/kind.py
Original file line number Diff line number Diff line change
Expand Up @@ -810,6 +810,13 @@ class EmbeddingModelRef(BaseModel):
model_namespace: str = Field("default", description="Embedding model namespace")


class CompleteEmbeddingModelRef(BaseModel):
"""Complete reference to an Embedding Model."""

model_name: str = Field(..., min_length=1, description="Embedding model name")
model_namespace: str = Field("default", description="Embedding model namespace")


class RetrieverRef(BaseModel):
"""Reference to a Retriever"""

Expand All @@ -835,23 +842,19 @@ def model_post_init(self, __context):


class RetrievalConfig(BaseModel):
"""Retrieval configuration for knowledge base

Note: retriever_name and embedding_config are optional to support knowledge bases
that don't use RAG (using kb_ls/kb_head tools instead for document exploration).
"""
"""Complete retrieval configuration for a RAG-enabled knowledge base."""

retriever_name: Optional[str] = Field(None, description="Retriever name")
retriever_name: str = Field(..., min_length=1, description="Retriever name")
retriever_namespace: str = Field("default", description="Retriever namespace")
embedding_config: Optional[EmbeddingModelRef] = Field(
None, description="Embedding model configuration"
embedding_config: CompleteEmbeddingModelRef = Field(
..., description="Embedding model configuration"
)
retrieval_mode: str = Field(
"vector", description="Retrieval mode: 'vector', 'keyword', or 'hybrid'"
)
top_k: int = Field(5, ge=1, le=10, description="Number of results to return")
score_threshold: float = Field(
0.7, ge=0.0, le=1.0, description="Minimum score threshold"
0.5, ge=0.0, le=1.0, description="Minimum score threshold"
)
hybrid_weights: Optional[HybridWeights] = Field(
None, description="Hybrid search weights"
Expand Down
56 changes: 53 additions & 3 deletions backend/app/schemas/knowledge.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
Pydantic schemas for knowledge base and document management.
"""

import logging
from datetime import datetime
from enum import Enum
from typing import Dict, List, Optional
from typing import Any, Dict, List, Literal, Optional

from pydantic import BaseModel, Field, field_validator, model_validator

Expand All @@ -28,6 +29,8 @@
from app.schemas.rag import SplitterConfig
from app.services.knowledge.splitter_config import normalize_splitter_config

logger = logging.getLogger(__name__)


class DocumentStatus(str, Enum):
"""Document status enumeration."""
Expand Down Expand Up @@ -94,6 +97,26 @@ class InitialMemberCreate(BaseModel):
)


class RetrievalConfigCreate(BaseModel):
"""Partial retrieval configuration accepted during knowledge base creation."""

retriever_name: Optional[str] = Field(None, description="Retriever name")
retriever_namespace: str = Field("default", description="Retriever namespace")
embedding_config: Optional[EmbeddingModelRef] = Field(
None, description="Embedding model configuration"
)
retrieval_mode: str = Field(
"vector", description="Retrieval mode: 'vector', 'keyword', or 'hybrid'"
)
top_k: int = Field(5, ge=1, le=10, description="Number of results to return")
score_threshold: float = Field(
0.5, ge=0.0, le=1.0, description="Minimum score threshold"
)
hybrid_weights: Optional[HybridWeights] = Field(
None, description="Hybrid search weights"
)


class KnowledgeBaseCreate(BaseModel):
"""Schema for creating a knowledge base."""

Expand All @@ -104,9 +127,13 @@ class KnowledgeBaseCreate(BaseModel):
"notebook",
description="Knowledge base type: 'notebook' (3-column layout with chat) or 'classic' (document list only)",
)
retrieval_config: Optional[RetrievalConfig] = Field(
retrieval_config: Optional[RetrievalConfigCreate] = Field(
None, description="Retrieval configuration"
)
rag_config_mode: Literal["auto", "disabled"] = Field(
"auto",
description="RAG configuration mode: auto-fill or disabled",
)
summary_enabled: bool = Field(
default=False,
description="Enable automatic summary generation for documents",
Expand Down Expand Up @@ -276,6 +303,27 @@ class KnowledgeBaseResponse(BaseModel):
created_at: datetime
updated_at: datetime

@staticmethod
def _normalize_retrieval_config_for_response(
config: Optional[Any], knowledge_base_id: int
) -> Optional[Dict[str, Any]]:
"""Drop historical incomplete retrieval configs before response validation."""
if not isinstance(config, dict):
return None

embedding_config = config.get("embedding_config") or {}
if not isinstance(embedding_config, dict):
embedding_config = {}

if config.get("retriever_name") and embedding_config.get("model_name"):
return config
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

logger.warning(
"Dropping incomplete retrievalConfig from knowledge base response: kb_id=%s",
knowledge_base_id,
)
return None

@classmethod
def from_kind(cls, kind, document_count: int = 0):
"""Create response from Kind object
Expand Down Expand Up @@ -318,7 +366,9 @@ def from_kind(cls, kind, document_count: int = 0):
namespace=kind.namespace,
kb_type=kb_type,
document_count=document_count,
retrieval_config=spec.get("retrievalConfig"),
retrieval_config=cls._normalize_retrieval_config_for_response(
spec.get("retrievalConfig"), kind.id
),
summary_enabled=spec.get("summaryEnabled", False),
summary_model_ref=summary_model_ref,
summary=summary,
Expand Down
152 changes: 105 additions & 47 deletions backend/app/services/knowledge/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,93 @@ def get_default_embedding_model(
"model_namespace": first.get("namespace", "default"),
}

def _resolve_retrieval_config(
self,
*,
db: Session,
user: User,
namespace: str,
retrieval_config: Optional[Dict[str, Any]],
rag_config_mode: Literal["auto", "disabled"] = "auto",
retriever_name: Optional[str] = None,
retriever_namespace: Optional[str] = None,
embedding_model_name: Optional[str] = None,
embedding_model_namespace: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Build a complete retrieval config, auto-filling missing core fields."""
if rag_config_mode == "disabled":
return None

resolved_config = dict(retrieval_config or {})
embedding_config = dict(resolved_config.get("embedding_config") or {})

retriever_name = resolved_config.get("retriever_name") or retriever_name
retriever_namespace = (
resolved_config.get("retriever_namespace") or retriever_namespace
)
embedding_model_name = (
embedding_config.get("model_name") or embedding_model_name
)
embedding_model_namespace = (
embedding_config.get("model_namespace") or embedding_model_namespace
)

if not retriever_name:
default_retriever = self.get_default_retriever(db, user.id, namespace)
if default_retriever:
retriever_name = default_retriever["retriever_name"]
retriever_namespace = default_retriever["retriever_namespace"]

if not embedding_model_name:
default_embedding = self.get_default_embedding_model(db, user.id, namespace)
if default_embedding:
embedding_model_name = default_embedding["model_name"]
embedding_model_namespace = default_embedding["model_namespace"]

if not retriever_name or not embedding_model_name:
logger.warning(
"[Orchestrator] Could not build retrieval_config: "
"retriever=%s, embedding=%s",
retriever_name,
embedding_model_name,
)
return None

logger.info(
"[Orchestrator] Built retrieval_config: retriever=%s, embedding=%s",
retriever_name,
embedding_model_name,
)
return self._build_complete_retrieval_config(
base_config=resolved_config,
retriever_name=retriever_name,
retriever_namespace=retriever_namespace,
embedding_model_name=embedding_model_name,
embedding_model_namespace=embedding_model_namespace,
)

def _build_complete_retrieval_config(
self,
*,
base_config: Dict[str, Any],
retriever_name: str,
retriever_namespace: Optional[str],
embedding_model_name: str,
embedding_model_namespace: Optional[str],
) -> Dict[str, Any]:
"""Build the only retrieval_config shape allowed to be persisted."""
resolved_config = dict(base_config)
resolved_config["retriever_name"] = retriever_name
resolved_config["retriever_namespace"] = retriever_namespace or "default"
resolved_config["embedding_config"] = {
"model_name": embedding_model_name,
"model_namespace": embedding_model_namespace or "default",
}
resolved_config.setdefault("retrieval_mode", "vector")
resolved_config.setdefault("top_k", 5)
resolved_config.setdefault("score_threshold", 0.5)
return resolved_config

def get_task_model_as_summary_model(
self,
db: Session,
Expand Down Expand Up @@ -959,6 +1046,7 @@ def create_knowledge_base(
namespace: str = "default",
kb_type: str = "notebook",
summary_enabled: bool = False,
rag_config_mode: Literal["auto", "disabled"] = "auto",
# REST API scenario: pass complete config
retrieval_config: Optional[Dict[str, Any]] = None,
# MCP scenario: auto-select or explicitly specify
Expand All @@ -973,11 +1061,11 @@ def create_knowledge_base(
"""
Create a knowledge base with auto-configuration support.

Supports two modes:
1. REST API mode: Pass complete retrieval_config dict
2. MCP mode: Auto-select or specify individual retriever/embedding params
Supports two RAG modes:
1. auto: Use a complete provided config or auto-select missing defaults.
2. disabled: Create a knowledge base without RAG retrieval config.

Auto-selection logic (when retrieval_config is None):
Auto-selection logic:
1. retriever: If not specified, auto-select using get_default_retriever()
2. embedding: If not specified, auto-select using get_default_embedding_model()
3. summary_model: If not specified and summary_enabled=True:
Expand All @@ -992,6 +1080,7 @@ def create_knowledge_base(
namespace: Namespace (default for personal, group name for group)
kb_type: Type (notebook or classic)
summary_enabled: Enable summary generation
rag_config_mode: RAG configuration mode
retrieval_config: Complete retrieval config dict (REST API mode)
retriever_name: Optional retriever name (MCP mode)
retriever_namespace: Optional retriever namespace (MCP mode)
Expand All @@ -1009,53 +1098,22 @@ def create_knowledge_base(
logger.info(
f"[Orchestrator] create_knowledge_base called: name={name}, namespace={namespace}, "
f"kb_type={kb_type}, summary_enabled={summary_enabled}, task_id={task_id}, "
f"rag_config_mode={rag_config_mode}, "
f"user_id={user.id}, has_retrieval_config={retrieval_config is not None}, "
f"has_summary_model_ref={summary_model_ref is not None}"
)

# Use provided retrieval_config or build one from individual params
resolved_retrieval_config = retrieval_config

if resolved_retrieval_config is None:
# MCP mode: auto-select or use provided individual params
# Auto-select retriever if not specified
if retriever_name is None:
default_retriever = self.get_default_retriever(db, user.id, namespace)
if default_retriever:
retriever_name = default_retriever["retriever_name"]
retriever_namespace = default_retriever["retriever_namespace"]

# Auto-select embedding model if not specified
if embedding_model_name is None:
default_embedding = self.get_default_embedding_model(
db, user.id, namespace
)
if default_embedding:
embedding_model_name = default_embedding["model_name"]
embedding_model_namespace = default_embedding["model_namespace"]

# Build retrieval_config if we have both retriever and embedding
if retriever_name and embedding_model_name:
resolved_retrieval_config = {
"retriever_name": retriever_name,
"retriever_namespace": retriever_namespace or "default",
"embedding_config": {
"model_name": embedding_model_name,
"model_namespace": embedding_model_namespace or "default",
},
"retrieval_mode": "vector",
"top_k": 5,
"score_threshold": 0.5,
}
logger.info(
f"[Orchestrator] Built retrieval_config: retriever={retriever_name}, "
f"embedding={embedding_model_name}"
)
else:
logger.warning(
f"[Orchestrator] Could not build retrieval_config: "
f"retriever={retriever_name}, embedding={embedding_model_name}"
)
resolved_retrieval_config = self._resolve_retrieval_config(
db=db,
user=user,
namespace=namespace,
retrieval_config=retrieval_config,
rag_config_mode=rag_config_mode,
retriever_name=retriever_name,
retriever_namespace=retriever_namespace,
embedding_model_name=embedding_model_name,
embedding_model_namespace=embedding_model_namespace,
)

# Auto-select summary model if summary_enabled and not specified
resolved_summary_model_ref = summary_model_ref
Expand Down
2 changes: 1 addition & 1 deletion backend/app/services/rag/runtime_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ def build_query_knowledge_base_configs_from_records(
),
retrieval_config=RuntimeRetrievalConfig(
top_k=retrieval_config.get("top_k", 20),
score_threshold=retrieval_config.get("score_threshold", 0.7),
score_threshold=retrieval_config.get("score_threshold", 0.5),
retrieval_mode=retrieval_mode,
vector_weight=(
hybrid_weights.get("vector_weight")
Expand Down
Loading
Loading