Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 37 additions & 6 deletions openrag/core/indexing/contextualize.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import asyncio
import logging
from collections.abc import Sequence
from contextlib import AbstractAsyncContextManager, nullcontext

from tqdm.asyncio import tqdm

Expand All @@ -27,7 +28,7 @@
logger = logging.getLogger(__name__)

DEFAULT_TIMEOUT_SECONDS = 30.0
DEFAULT_MAX_CONCURRENT = 4
DEFAULT_BATCH_SIZE = 4


class ChunkContextualizer:
Expand All @@ -39,14 +40,17 @@ def __init__(
system_prompt: str,
*,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
max_concurrent: int = DEFAULT_MAX_CONCURRENT,
semaphore: asyncio.Semaphore | None = None,
batch_size: int = DEFAULT_BATCH_SIZE,
llm_semaphore: AbstractAsyncContextManager[object] | None = None,
):
self._llm = llm
self._system_prompt = system_prompt
self._timeout = timeout_seconds
self._batch_size = max(1, max_concurrent)
self._semaphore = semaphore or asyncio.Semaphore(self._batch_size)
self._batch_size = max(1, batch_size)
# Optional cluster-wide LLM gate, injected by the caller (e.g. the Ray
# "llmSemaphore"); wraps each chat call so contextualization shares the
# global LLM concurrency budget. No-op when not supplied.
self._semaphore = llm_semaphore or nullcontext()

async def _generate_context(
self,
Expand All @@ -66,7 +70,8 @@ async def _generate_context(
)
async with self._semaphore:
try:
return await asyncio.wait_for(self._llm.chat(messages), timeout=self._timeout)
response = await asyncio.wait_for(self._llm.chat(messages), timeout=self._timeout)
return _chat_response_text(response)
except TimeoutError:
logger.warning("LLM timeout contextualizing chunk (filename=%s)", filename)
return ""
Expand Down Expand Up @@ -136,3 +141,29 @@ async def contextualize(
except (TimeoutError, OSError, RuntimeError, ValueError) as exc:
logger.warning("Error contextualizing chunks from %s: %s", filename, exc)
return chunks


def _chat_response_text(response: object) -> str:
"""Extract the assistant text from an LLM ``chat`` result.

Accepts a raw string or an OpenAI-style response dict
(``choices[0].message.content``, ``choices[0].text``, or a top-level
``content``); returns ``""`` for any unrecognized shape.
"""
if isinstance(response, str):
return response
if not isinstance(response, dict):
return ""

choices = response.get("choices")
if isinstance(choices, list) and choices:
first = choices[0]
if isinstance(first, dict):
message = first.get("message")
if isinstance(message, dict) and isinstance(message.get("content"), str):
return message["content"]
if isinstance(first.get("text"), str):
return first["text"]

content = response.get("content")
return content if isinstance(content, str) else ""
1 change: 1 addition & 0 deletions openrag/di/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ async def initialize(self) -> None:
await self._initialize_step("seeding model endpoints", self.model_endpoint_service.seed_defaults)
await self._initialize_step("loading model endpoints", self.model_endpoint_service.load_all)
await self._initialize_step("seeding pipeline presets", self.preset_service.seed_defaults)
await self._initialize_step("syncing preset env toggles", self.preset_service.sync_env_toggles)
await self._initialize_step("loading pipeline presets", self.preset_service.load_all)
await self._initialize_step("ensuring default partition", self.partition_service.seed_default_partition)
await self._initialize_step("loading partition configs", self.partition_service.load_partitions)
Expand Down
52 changes: 44 additions & 8 deletions openrag/services/orchestrators/preset_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,16 @@
_VALID_PRESET_TYPES = frozenset({"indexation", "retrieval"})

_DEFAULT_SEEDS: dict[str, dict[str, dict[str, Any]]] = {
# ``enable_contextualization`` is intentionally omitted from the ``default``
# indexation preset below — it is injected at seed time from the global
# ``chunker.contextual_retrieval`` (``CONTEXTUAL_RETRIEVAL``) toggle in
# _finalize_seed, so the env flag still drives contextualization on fresh
# deployments. Named presets (legal/finance) keep their explicit choice.
"indexation": {
"default": {
"chunking": {"name": "recursive_splitter", "chunk_size": 512, "chunk_overlap_rate": 0.2},
"parsing_strategy": "marker",
"enable_image_captioning": True,
"enable_contextualization": False,
"enable_entity_extraction": True,
"enable_topic_tagging": True,
},
Expand Down Expand Up @@ -105,21 +109,53 @@ async def seed_defaults(self) -> None:
if existing:
continue
for name, config in presets.items():
await self._repo.upsert(name, preset_type, self._finalize_seed(preset_type, config))
await self._repo.upsert(name, preset_type, self._finalize_seed(name, preset_type, config))
logger.info(f"Seeded default {preset_type} preset '{name}'.")

def _finalize_seed(self, preset_type: str, config: dict[str, Any]) -> dict[str, Any]:
"""Overlay env-derived values onto a static default seed.
def _finalize_seed(self, name: str, preset_type: str, config: dict[str, Any]) -> dict[str, Any]:
"""Overlay env-derived global toggles onto a static default seed.

Two top-level kill-switches are folded into the seeded presets so a
fresh deployment honours the env flags without admin action:

Default retrieval presets inherit the global reranker kill-switch
(``reranker.enabled``): when no reranker is available the default
presets ship with ``enable_reranker=False`` so partition resolution
never tries to build a reranker against a missing/unreachable endpoint.
* every retrieval preset inherits ``reranker.enabled`` (``enable_reranker``)
so a deployment without a reranker never builds one against a
missing/unreachable endpoint;
* the ``default`` indexation preset inherits ``chunker.contextual_retrieval``
(``CONTEXTUAL_RETRIEVAL``) as ``enable_contextualization`` so the global
toggle still drives contextualization. Named presets keep the explicit
value carried in the seed.
"""
if preset_type == "retrieval":
return {**config, "enable_reranker": self._config.reranker.enabled}
if preset_type == "indexation" and name == "default":
return {**config, "enable_contextualization": self._config.chunker.contextual_retrieval}
return config

async def sync_env_toggles(self) -> None:
"""Re-apply env-derived global toggles onto the ``default`` preset.

Runs on every boot (unlike :meth:`seed_defaults`, which only fires
against an empty table), so a changed ``CONTEXTUAL_RETRIEVAL`` env flag
takes effect after a restart without manual edits. Only the ``default``
indexation preset's ``enable_contextualization`` is touched, and only
when it actually differs — named presets (legal/finance) keep their
explicit values. ``load_all``/``load_partitions`` run afterwards in the
startup sequence, so the refreshed value reaches the in-memory caches.
"""
row = await self._repo.get("default", "indexation")
if row is None:
return
desired = self._config.chunker.contextual_retrieval
config = row["config"]
if config.get("enable_contextualization") == desired:
return
await self._repo.upsert("default", "indexation", {**config, "enable_contextualization": desired})
Comment thread
hedhoud marked this conversation as resolved.
Resolved
logger.info(
"Synced 'default' indexation preset to CONTEXTUAL_RETRIEVAL env.",
enable_contextualization=desired,
)

async def load_all(self) -> None:
"""Fetch all presets from DB, rebuild config.presets dicts atomically.

Expand Down
98 changes: 97 additions & 1 deletion openrag/services/workers/indexer_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import ray
from services.workers.indexer_actor import IndexerWorker

from openrag.core.config.root import Settings


@ray.remote
class IndexerPool:
Expand All @@ -28,6 +30,7 @@ def __init__(self) -> None:
parser = DocSerializerBridgeParser(config=cfg)
chunker = _build_chunker(cfg)
embedder_factory = _build_embedder_factory(cfg)
contextualizer_factory = _build_contextualizer_factory(cfg)

embed_cfg = cfg.embedder
embedder = embedder_registry.create(
Expand All @@ -49,6 +52,7 @@ def __init__(self) -> None:
vector_store=self._vector_store,
chunker_factory=_build_chunker_from_config,
embedder_factory=embedder_factory,
contextualizer_factory=contextualizer_factory,
)
rdb_cfg = cfg.rdb.model_copy(update={"database": f"partitions_for_collection_{cfg.vectordb.collection_name}"})
self._catalog_store = PostgresStore(rdb_cfg, run_migrations=False)
Expand Down Expand Up @@ -135,7 +139,7 @@ def _build_chunker_from_config(chunker_config: Any) -> Any:
return _build_chunker(SimpleNamespace(chunker=chunker_config))


def _build_embedder_factory(cfg: Any) -> Any:
def _build_embedder_factory(cfg: Settings) -> Any:
if not getattr(cfg.models, "embedder", None):
return None

Expand Down Expand Up @@ -169,4 +173,96 @@ def factory(name: str = "default") -> Any:
return factory


def _build_contextualizer_factory(cfg: Settings) -> Any:
"""Build a cached factory yielding ``ChunkContextualizer`` instances.

Each requested LLM name is resolved against the named-endpoint registry
(``cfg.models.llm``), falling back to the global ``cfg.llm`` block for the
``default`` name. Returns ``None`` when no LLM is configured. Every
contextualizer shares the cluster-wide ``llmSemaphore`` so contextualization
LLM calls obey the same global concurrency limit as query-time calls.
"""
import services.inference.ollama_client # noqa: F401
import services.inference.vllm_client # noqa: F401
from core.indexing.contextualize import ChunkContextualizer
from core.llm import llm_registry
from core.prompts import load_template_by_key
from services.inference.distributed_semaphore import DistributedSemaphore

named_llms = getattr(getattr(cfg, "models", None), "llm", {}) or {}
fallback_cfg = _global_llm_endpoint_config(cfg)
if not named_llms and fallback_cfg is None:
return None

system_prompt = load_template_by_key(cfg.paths.prompts_dir, cfg.prompts, "chunk_contextualizer")
# Rate-limit contextualization LLM calls with the cluster-wide "llmSemaphore"
# (the same one get_llm_semaphore() / bootstrap.py manage) so a large indexing
# job can't flood the vLLM endpoint. Built directly from config to avoid
# importing services.inference.runtime, which eagerly constructs a LangDetector.
llm_semaphore = DistributedSemaphore(name="llmSemaphore", max_concurrent_ops=cfg.semaphore.llm_semaphore)
cache: dict[str, ChunkContextualizer] = {}
lock = threading.Lock()

def factory(name: str = "default") -> ChunkContextualizer:
if name in cache:
return cache[name]
with lock:
if name in cache:
return cache[name]
model_cfg = named_llms.get(name)
if model_cfg is None:
if name == "default" and fallback_cfg is not None:
model_cfg = fallback_cfg
else:
raise KeyError(f"Unknown llm '{name}'. Available: {list(named_llms)}")
impl_kwargs = {key: value for key, value in model_cfg.extra.items() if key != "implementation"}
impl = model_cfg.extra.get("implementation", "vllm")
llm = llm_registry.create(
impl,
endpoint=model_cfg.endpoint,
model_name=model_cfg.model_name,
timeout=model_cfg.timeout,
**impl_kwargs,
)
contextualizer = ChunkContextualizer(
llm,
system_prompt,
timeout_seconds=cfg.chunker.contextualization_timeout,
batch_size=cfg.chunker.max_concurrent_contextualization,
llm_semaphore=llm_semaphore,
)
cache[name] = contextualizer
return contextualizer

return factory


def _global_llm_endpoint_config(cfg: Any) -> Any | None:
"""Adapt the legacy/global ``cfg.llm`` block into a ``ModelEndpointConfig``.

Returns ``None`` when the global LLM endpoint or model is unset, signalling
that no ``default`` contextualizer can be built from the global config.
"""
from core.config.model_endpoints import ModelEndpointConfig

llm_cfg = getattr(cfg, "llm", None)
endpoint = getattr(llm_cfg, "base_url", "")
model_name = getattr(llm_cfg, "model", "")
if not endpoint or not model_name:
return None
extra = {
"implementation": "vllm",
"api_key": getattr(llm_cfg, "api_key", ""),
"temperature": getattr(llm_cfg, "temperature", 0.1),
"max_retries": getattr(llm_cfg, "max_retries", 2),
"logprobs": getattr(llm_cfg, "logprobs", True),
}
return ModelEndpointConfig(
endpoint=endpoint,
model_name=model_name,
timeout=getattr(llm_cfg, "timeout", 60),
extra=extra,
)


__all__ = ["IndexerPool", "build_indexer_pool"]
57 changes: 57 additions & 0 deletions tests/unit/core/indexing/test_contextualize.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
from __future__ import annotations

import pytest
from core.indexing.contextualize import ChunkContextualizer
from core.models.chunk import Chunk


class DictLLM:
async def chat(self, messages, **kwargs):
return {"choices": [{"message": {"content": "document-level context"}}]}


@pytest.mark.asyncio
async def test_contextualizer_accepts_openai_style_chat_response():
contextualizer = ChunkContextualizer(DictLLM(), "System prompt")

result = await contextualizer.contextualize([Chunk(id="c1", text="chunk body", partition="p")])

assert result[0].context == "document-level context"
assert result[0].content == "chunk body"


class _GatingLLM:
"""LLM stub that records whether the LLM gate was held during ``chat``."""

def __init__(self, gate_held: list[bool]):
self._gate_held = gate_held

async def chat(self, messages, **kwargs):
self._gate_held.append(_TrackingGate.depth > 0)
return {"choices": [{"message": {"content": "ctx"}}]}


class _TrackingGate:
"""Async context manager standing in for the distributed LLM semaphore."""

depth = 0

async def __aenter__(self):
type(self).depth += 1
return self

async def __aexit__(self, *exc):
type(self).depth -= 1


@pytest.mark.asyncio
async def test_contextualizer_holds_llm_semaphore_around_chat():
_TrackingGate.depth = 0
gate_held: list[bool] = []
contextualizer = ChunkContextualizer(_GatingLLM(gate_held), "System prompt", llm_semaphore=_TrackingGate())

await contextualizer.contextualize([Chunk(id="c1", text="chunk body", partition="p")])

# The injected gate was entered for the chat call and released afterwards.
assert gate_held == [True]
assert _TrackingGate.depth == 0
7 changes: 7 additions & 0 deletions tests/unit/di/test_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ async def initialize(self):
)
c._preset_service = SimpleNamespace(
seed_defaults=lambda: _async_call(calls, "preset.seed"),
sync_env_toggles=lambda: _async_call(calls, "preset.sync"),
load_all=lambda: _async_call(calls, "preset.load"),
)
c._partition_service = SimpleNamespace(
Expand All @@ -256,6 +257,7 @@ async def initialize(self):
"endpoint.seed",
"endpoint.load",
"preset.seed",
"preset.sync",
"preset.load",
"partition.seed",
"partition.load",
Expand Down Expand Up @@ -669,6 +671,10 @@ async def seed_defaults(self):
"""Record preset seeding."""
calls.append("preset.seed")

async def sync_env_toggles(self):
"""Record preset env re-sync."""
calls.append("preset.sync")

async def load_all(self):
"""Record preset loading."""
calls.append("preset.load")
Expand Down Expand Up @@ -699,6 +705,7 @@ async def load_partitions(self):
"endpoint.seed",
"endpoint.load",
"preset.seed",
"preset.sync",
"preset.load",
"partition.seed",
"partition.load",
Expand Down
Loading
Loading