feat(prompts): DB-backed prompt library with per-preset and per-partition selection

docs/content/docs/documentation/API.mdx

@@ -513,6 +513,73 @@ DELETE /presets/{preset_type}/{name}
 
 ---
 
+### 📝 Prompt Library
+
+Every prompt the pipeline sends to a model is a stored, editable row rather than a bundled file. On first boot each type is seeded from its bundled template as that type's **default**; an admin can add named variants and select one per preset or per partition.
+
+All routes are prefixed with **`/prompts`** and require the **admin** role. `prompt_type` is one of `sys_prompt` | `spoken_style_answer` | `query_contextualizer` | `chunk_contextualizer` | `image_captioning` | `hyde` | `multi_query` | `topic_tagger`.
+
+**Resolution order** for a given type: the name selected for the request → the type's global default → the bundled template. A selection naming a prompt that no longer exists falls back to the default rather than failing.
+
+**Where a prompt is selected** — each setting lives with the thing it configures:
+
+| Prompt type | Selected on |
+|---|---|
+| `sys_prompt`, `spoken_style_answer` | partition — `generation_prompt_names` |
+| `query_contextualizer`, `hyde`, `multi_query` | retrieval preset — `*_prompt_name` |
+| `chunk_contextualizer`, `image_captioning`, `topic_tagger` | indexation preset — `*_prompt_name` |
+
+#### Create a prompt
+```http
+POST /prompts/
+```
+**Body:** `prompt_type`, `name`, `content`, `is_default` (default `false`). Returns `201 Created`, or `409` if that `(prompt_type, name)` already exists.
+
+Types rendered as templates (`sys_prompt`, `spoken_style_answer`, `query_contextualizer`, `hyde`, `multi_query`) accept only their own **plain** `{placeholders}` — no conversion (`!r`), format spec (`:>10`) or attribute access — and a violation returns `422` at write time rather than failing later at render. Escape a literal brace as `{{` / `}}`. The remaining types are sent to the model verbatim, so any text is valid.
+
+```bash frame="none"
+curl -X POST http://localhost:8080/prompts/ \
+  -H "Authorization: Bearer YOUR_AUTH_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt_type": "sys_prompt",
+    "name": "legal-assistant",
+    "content": "Answer strictly from the context.\n{context}\nToday is {current_date}."
+  }'
+```
+
+#### List / Get / Update / Delete
+```http
+GET    /prompts/                 # ?prompt_type= to filter, ?offset= &limit= to page (limit ≤ 500)
+GET    /prompts/{prompt_id}
+PATCH  /prompts/{prompt_id}      # any of name, content, is_default
+DELETE /prompts/{prompt_id}      # 204 No Content; refused for a type's current default
+```
+List entries carry `used_by` — the number of partitions that resolve to that prompt, counting those that fall back to it as the default.
+
+#### Promote to default for its type
+```http
+PUT /prompts/{prompt_id}/default
+```
+Clears the previous default for that type and promotes this one, atomically.
+
+#### Selecting a prompt for a partition
+```http
+PATCH /partition/{partition}
+```
+Send `generation_prompt_names`, a map of `{prompt_type: name}` restricted to `sys_prompt` and `spoken_style_answer`. Each name must exist, or the request returns `422`. Send `{}` to clear the selection and fall back to the defaults.
+
+```bash frame="none"
+curl -X PATCH http://localhost:8080/partition/my-partition \
+  -H "Authorization: Bearer YOUR_AUTH_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"generation_prompt_names": {"sys_prompt": "legal-assistant"}}'
+```
+
+Preset-scoped prompts are selected the same way, by putting the `*_prompt_name` field in the preset's `config` (see [Pipeline Presets](#-pipeline-presets)).
+
+---
+
 ### 🔌 Model Endpoints
 
 A registry of named inference endpoints (embedder, reranker, LLM, VLM) that partitions and presets can point at, so operators can manage and switch inference backends at runtime instead of via `.env`. Stored API keys are **redacted** in every response and only returned through the explicit reveal action below.

openrag/api/main.py

@@ -46,6 +46,7 @@
 from api.routers.admin.monitoring import router as monitoring_router
 from api.routers.admin.partitions import router as partition_router
 from api.routers.admin.presets import router as presets_router
+from api.routers.admin.prompts import router as prompts_router
 from api.routers.admin.tools import router as tools_router
 from api.routers.admin.users import router as users_router
 from api.routers.admin.workspaces import router as workspaces_router
@@ -117,6 +118,7 @@ class Tags(Enum):
     PARTITION = "Partitions & files"
     MODEL_ENDPOINTS = "Model Endpoints"
     PRESETS = "Presets"
+    PROMPTS = "Prompts"
     QUEUE = "Queue management"
     ACTORS = "Ray Actors"
     USERS = "User management"
@@ -357,6 +359,7 @@ def get_config():
 app.include_router(partition_router, prefix="/partition", tags=[Tags.PARTITION])
 app.include_router(model_endpoints_router, prefix="/model-endpoints", tags=[Tags.MODEL_ENDPOINTS])
 app.include_router(presets_router, prefix="/presets", tags=[Tags.PRESETS])
+app.include_router(prompts_router, prefix="/prompts", tags=[Tags.PROMPTS])
 app.include_router(queue_router, prefix="/queue", tags=[Tags.QUEUE])
 app.include_router(actors_router, prefix="/actors", tags=[Tags.ACTORS])
 app.include_router(users_router, prefix="/users", tags=[Tags.USERS])

openrag/api/routers/admin/prompts.py

@@ -0,0 +1,77 @@
+"""Admin routes for the DB prompt library.
+
+Transport-only: auth, request validation, and response shaping live here;
+persistence and resolution are delegated to ``PromptService`` from the DI
+container. Per-partition assignment routes live alongside the other partition
+sub-resources in ``partitions.py``.
+"""
+
+from api.dependencies.auth import require_admin
+from api.schemas.admin.prompt_schemas import (
+    CreatePromptRequest,
+    PromptResponse,
+    PromptTypeName,
+    UpdatePromptRequest,
+)
+from di.providers import get_prompt_service
+from fastapi import APIRouter, Depends, Query, Response, status
+
+router = APIRouter(dependencies=[Depends(require_admin)])
+
+
+@router.post("/", response_model=PromptResponse, status_code=status.HTTP_201_CREATED)
+async def create_prompt(
+    body: CreatePromptRequest,
+    service=Depends(get_prompt_service),
+):
+    """Add a prompt to the library."""
+    return await service.create_prompt(**body.model_dump())
+
+
+@router.get("/", response_model=list[PromptResponse])
+async def list_prompts(
+    prompt_type: PromptTypeName | None = None,
+    offset: int = Query(0, ge=0),
+    limit: int = Query(100, ge=1, le=500),
+    service=Depends(get_prompt_service),
+):
+    """List library prompts (optionally by type), each with an override count."""
+    return await service.list_prompts(prompt_type=prompt_type, offset=offset, limit=limit)
+
+
+@router.get("/{prompt_id}", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str,
+    service=Depends(get_prompt_service),
+):
+    """Return one library prompt."""
+    return await service.get_prompt(prompt_id)
+
+
+@router.patch("/{prompt_id}", response_model=PromptResponse)
+async def update_prompt(
+    prompt_id: str,
+    body: UpdatePromptRequest,
+    service=Depends(get_prompt_service),
+):
+    """Edit a prompt's name/content and/or promote it to default."""
+    return await service.update_prompt(prompt_id, **body.model_dump(exclude_unset=True))
+
+
+@router.put("/{prompt_id}/default", response_model=PromptResponse)
+async def set_prompt_default(
+    prompt_id: str,
+    service=Depends(get_prompt_service),
+):
+    """Promote a prompt to the default for its type."""
+    return await service.set_default(prompt_id)
+
+
+@router.delete("/{prompt_id}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_prompt(
+    prompt_id: str,
+    service=Depends(get_prompt_service),
+):
+    """Delete a library prompt (rejected if it is the current default)."""
+    await service.delete_prompt(prompt_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

openrag/api/schemas/admin/partition_schemas.py

@@ -62,6 +62,24 @@ class UpdatePartitionRequest(BaseModel):
     retrieval_preset: str | None = None
     chat_history_depth: int | None = Field(default=None, ge=1)
     chat_llm: str | None = None
+    # {prompt_type: library_prompt_name} for this partition's generation prompts.
+    # Keys are restricted to the generation types; ``{}`` clears all overrides.
+    generation_prompt_names: dict[str, str] | None = None
+
+    @field_validator("generation_prompt_names")
+    @classmethod
+    def validate_generation_prompt_names(cls, value: dict[str, str] | None, info: ValidationInfo) -> dict[str, str]:
+        value = _reject_explicit_null(info.field_name, value)
+        # query_contextualizer is a query-side prompt selected on the retrieval
+        # preset, not a partition generation prompt (see RetrievalPipelineConfig).
+        allowed = {"sys_prompt", "spoken_style_answer"}
+        bad = set(value) - allowed
+        if bad:
+            raise ValueError(f"generation_prompt_names keys must be one of {sorted(allowed)}; got {sorted(bad)}")
+        for k, v in value.items():
+            if not isinstance(v, str) or not v.strip():
+                raise ValueError(f"generation_prompt_names['{k}'] must be a non-empty prompt name")
+        return value
 
     @field_validator("embedder", "indexation_preset", "retrieval_preset")
     @classmethod
@@ -113,6 +131,7 @@ class PartitionDetailResponse(BaseModel):
     document_count: int = 0
     chat_history_depth: int = 4
     chat_llm: str | None = None
+    generation_prompt_names: dict[str, str] = Field(default_factory=dict)
 
 
 __all__ = [

openrag/api/schemas/admin/prompt_schemas.py

@@ -0,0 +1,98 @@
+"""Admin schemas for the DB prompt library."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator, model_validator
+
+# The managed prompt types (mirrors core.models.prompt.PromptType). Declaring
+# them as a Literal makes FastAPI reject unknown types at the transport edge
+# (422) and renders the enum in the OpenAPI schema.
+PromptTypeName = Literal[
+    "sys_prompt",
+    "query_contextualizer",
+    "chunk_contextualizer",
+    "image_captioning",
+    "hyde",
+    "multi_query",
+    "spoken_style_answer",
+    "topic_tagger",
+]
+
+
+def _require_non_empty(field_name: str, value: str) -> str:
+    value = value.strip()
+    if not value:
+        raise ValueError(f"{field_name} must be non-empty")
+    return value
+
+
+class CreatePromptRequest(BaseModel):
+    """Request body for adding a prompt to the library."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    prompt_type: PromptTypeName
+    name: str
+    content: str
+    is_default: bool = False
+
+    @field_validator("name", "content")
+    @classmethod
+    def validate_non_empty(cls, value: str, info: ValidationInfo) -> str:
+        return _require_non_empty(info.field_name, value)
+
+
+class UpdatePromptRequest(BaseModel):
+    """Request body for editing a prompt and/or promoting it to default."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str | None = None
+    content: str | None = None
+    is_default: bool | None = None
+
+    @field_validator("content")
+    @classmethod
+    def validate_content(cls, value: str | None, info: ValidationInfo) -> str | None:
+        if value is None:
+            raise ValueError(f"{info.field_name} cannot be null")
+        return _require_non_empty("content", value)
+
+    @field_validator("name")
+    @classmethod
+    def validate_name(cls, value: str | None, info: ValidationInfo) -> str | None:
+        if value is None:
+            raise ValueError(f"{info.field_name} cannot be null")
+        return _require_non_empty(info.field_name, value)
+
+    @model_validator(mode="after")
+    def require_at_least_one_update(self) -> UpdatePromptRequest:
+        if not any(getattr(self, f) is not None for f in ("name", "content", "is_default")):
+            raise ValueError("at least one field must be provided")
+        return self
+
+
+class PromptResponse(BaseModel):
+    """A stored library prompt."""
+
+    id: str
+    prompt_type: str
+    name: str
+    content: str
+    is_default: bool
+    created_at: datetime
+    updated_at: datetime
+    # Number of partitions/presets referencing this prompt by name. Populated by
+    # the list endpoint; 0 on single-item responses where usage isn't computed.
+    used_by: int = 0
+
+
+__all__ = [
+    "CreatePromptRequest",
+    "PromptResponse",
+    "PromptTypeName",
+    "UpdatePromptRequest",
+]

openrag/core/config/indexation_pipeline.py

@@ -42,9 +42,12 @@ class IndexationPipelineConfig(BaseModel):
     enable_metadata_extraction: bool = True
     metadata_extraction_llm: str | None = None
 
-    # Prompt name overrides (None = use active prompt for the partition)
-    vlm_caption_prompt_name: str | None = None
+    # Prompt selection: name a library prompt for this preset's enrichment
+    # stages (None = fall back to the type's global default, then the disk seed).
+    # Resolved per file in the indexer via PromptService.resolve_prompt.
     contextualization_prompt_name: str | None = None
+    image_captioning_prompt_name: str | None = None
+    topic_tagging_prompt_name: str | None = None
 
     # Entity extraction
     enable_entity_extraction: bool = True

openrag/core/config/retrieval_pipeline.py

@@ -29,5 +29,16 @@ class RetrievalPipelineConfig(BaseModel):
     include_ancestors: bool = True
     rrf_k: int = Field(default=60, gt=0, le=1000)  # Reciprocal Rank Fusion constant
 
+    # Prompt selection: name a library prompt for this preset's query-side
+    # prompts (None = the type's global default, then the disk seed). hyde /
+    # multi_query drive the query-expansion strategies (resolved per request in
+    # RetrievalService); query_contextualizer rewrites the user's query before
+    # retrieval (resolved in QueryService.generate_query). All three are
+    # query-side concerns, so they live on the retrieval preset rather than the
+    # partition's generation prompts.
+    hyde_prompt_name: str | None = None
+    multi_query_prompt_name: str | None = None
+    query_contextualizer_prompt_name: str | None = None
+
 
 __all__ = ["RetrievalPipelineConfig"]

openrag/core/indexing/contextualize.py

@@ -59,9 +59,10 @@ async def _generate_context(
         current_chunk: Chunk,
         filename: str,
         lang: str,
+        system_prompt: str,
     ) -> str:
         messages = build_messages(
-            system_prompt=self._system_prompt,
+            system_prompt=system_prompt,
             filename=filename,
             first_chunks_text=[c.text for c in first_chunks],
             prev_chunks_text=[c.text for c in prev_chunks],
@@ -85,6 +86,7 @@ async def contextualize(
         *,
         filename: str = "",
         lang: str = "en",
+        system_prompt: str | None = None,
     ) -> list[Chunk]:
         """Return new chunks with context prepended to ``text``.
 
@@ -100,6 +102,10 @@ async def contextualize(
         if not chunks:
             return []
 
+        # A per-call override (the DB-resolved prompt for this file's partition)
+        # wins over the instance default baked in at construction.
+        effective_prompt = system_prompt or self._system_prompt
+
         try:
             first_chunks = chunks[:2]
             contexts: list[str] = []
@@ -114,6 +120,7 @@ async def contextualize(
                         current_chunk=chunks[i],
                         filename=filename,
                         lang=lang,
+                        system_prompt=effective_prompt,
                     )
                     for i in range(start, end)
                 ]