Delegate embedding model discovery to contextual-orchestrator

CHANGELOG.d/2.20.1-orchestrator-temperature-negotiation.md

@@ -0,0 +1,8 @@
+# 2.20.1 — Provider temperature capability negotiation
+
+- Compose now pins ContextualWisdomLab/contextual-orchestrator#791 so providers
+  that accept only their default sampling temperature can retry the same
+  request without the unsupported optional field, alongside the orchestrator's
+  embedding-capability selection contract.
+- The pin remains an immutable upstream commit and does not copy provider
+  behavior or credentials into LineageWeave.

CHANGELOG.d/2.20.2-orchestrator-owned-embedding.md

@@ -0,0 +1,5 @@
+# 2.20.2
+
+- Delegate embedding-model selection to contextual-orchestrator, retain the
+  resolved model as database provenance, and keep provider model selectors out
+  of the LineageWeave backend process.

backend/app/config.py

@@ -129,7 +129,9 @@ def load_settings() -> Settings:
         ],
         orchestrator_base_url=os.environ.get("ORCHESTRATOR_BASE_URL", ""),
         orchestrator_api_key=os.environ.get("ORCHESTRATOR_API_KEY", ""),
-        embedding_model=os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip(),
+        # contextual-orchestrator owns embedding model discovery. Keep the
+        # empty compatibility value for older callers; never read a selector.
+        embedding_model="",
         valkey_url=os.environ.get("VALKEY_URL", "redis://localhost:16379/0"),
         searxng_base_url=os.environ.get("SEARXNG_BASE_URL", ""),
         tepp_transport_url=os.environ.get("TEPP_TRANSPORT_URL", ""),

backend/app/post_content_queue.py

@@ -48,7 +48,7 @@ async def post_content_is_complete(
     conn: asyncpg.Connection,
     post_id: str,
     *,
-    embedding_model_code: str,
+    embedding_model_code: str | None,
     require_structure: bool = False,
 ) -> bool:
     """Require configured semantic, structure, and region evidence before ready."""
@@ -61,8 +61,35 @@ async def post_content_is_complete(
                         where unit.post_id = $1
                    )
                and (
-                   $2 = ''
-                       or (
+                   (
+                       nullif($2::text, '') is null
+                       and (
+                           not exists(
+                               select 1
+                                 from post_content_unit unit
+                                 left join post_content_embedding embedding
+                                   on embedding.post_content_unit_id = unit.post_content_unit_id
+                                where unit.post_id = $1
+                                  and embedding.post_content_embedding_id is null
+                           )
+                           and not exists(
+                               select 1
+                                 from post_content_unit unit
+                                 join post_content_image image
+                                   on image.post_content_unit_id = unit.post_content_unit_id
+                                 join post_content_image_region region
+                                   on region.post_content_image_id = image.post_content_image_id
+                                 left join post_content_image_region_embedding embedding
+                                   on embedding.post_content_image_region_id = region.post_content_image_region_id
+                                where unit.post_id = $1
+                                  and region.description_status_code = 'described'
+                                  and embedding.post_content_image_region_embedding_id is null
+                           )
+                       )
+                   )
+                   or (
+                       nullif($2::text, '') is not null
+                       and (
                            not exists(
                                select 1
                                  from post_content_unit unit
@@ -88,6 +115,7 @@ async def post_content_is_complete(
                            )
                        )
                    )
+                   )
                and (
                        not $3::boolean
                        or not exists(

docker-compose.yml

@@ -161,9 +161,6 @@ services:
       # LLM_GATEWAY_API_KEY in the orchestrator's private env file.
       ORCHESTRATOR_BASE_URL: ${ORCHESTRATOR_BASE_URL:-http://orchestrator:8000}
       ORCHESTRATOR_API_KEY: ${ORCHESTRATOR_API_KEY:-${CONTEXTUAL_ORCHESTRATOR_TOKEN:-lineageweave-orchestrator-dev-only}}
-      # This is a non-secret embedding contract. Provider credentials remain
-      # private to contextual-orchestrator and are never injected into backend.
-      LLM_GATEWAY_EMBEDDING_MODEL: ${LLM_GATEWAY_EMBEDDING_MODEL:-text-embedding-3-large}
       SEARXNG_BASE_URL: http://searxng:8080
       TEPP_TRANSPORT_URL: ${TEPP_TRANSPORT_URL:-}
       TEPP_API_KEY: ${TEPP_API_KEY:-}

docker/contextual-orchestrator/Dockerfile

@@ -3,10 +3,11 @@ FROM python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a
 WORKDIR /app
 
 # Reuse the upstream implementation without copying it into LineageWeave.
-# Pin the runtime to PR #765's pushed commit until that protected PR merges;
+# Pin the runtime to ContextualWisdomLab/contextual-orchestrator#791's pushed
+# integration commit until its protected dependency stack merges;
 # this keeps model selection, structured synthesis, and reasoning policy in
 # contextual-orchestrator itself.
-ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/7df051a.tar.gz /tmp/contextual-orchestrator.tar.gz
+ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/0246e91f107c8dbb82eb0b421d0aeb0f5fe21ecf.tar.gz /tmp/contextual-orchestrator.tar.gz
 RUN mkdir /tmp/contextual-orchestrator \
     && tar -xzf /tmp/contextual-orchestrator.tar.gz --strip-components=1 -C /tmp/contextual-orchestrator \
     && cp -R /tmp/contextual-orchestrator/contextual_orchestrator /app/contextual_orchestrator \

docs/adr/0030-external-llm-gateway-environment.md

@@ -21,10 +21,10 @@ These credentials are not interchangeable. The first is the local service
 boundary; the second is the provider credential.
 
 The backend must not load `~/.env` wholesale. Compose injects the provider
-credential and URL only into contextual-orchestrator; the backend receives
-only its internal orchestrator credential and the non-secret embedding model
-identifier. This prevents an unrelated application process from holding the
-provider secret while preserving the single LLM/Vision boundary.
+credential, URL, and embedding-agent allowlist only into
+contextual-orchestrator; the backend receives only its internal orchestrator
+credential. This prevents an unrelated application process from holding a
+provider secret or selector while preserving the single LLM/Vision boundary.
 
 ## Decision
 

docs/adr/0083-orchestrator-runtime-commit-pin.md

@@ -10,14 +10,19 @@ The orchestrator's `auto` reasoning mode is an internal routing decision and
 must not be forwarded as an upstream provider `reasoning_effort` value. The
 runtime also must discover provider models from the configured gateway rather
 than requiring `LLM_GATEWAY_MODEL`, and structured requests must remain
-multi-agent.
+multi-agent. Some provider deployments accept only their default sampling
+temperature; retrying the same rejected value cannot recover that capability
+mismatch.
 
 ## Decision
 
 `docker/contextual-orchestrator/Dockerfile` pins the downloaded archive to
-commit `6db772e`, the pushed head of contextual-orchestrator PR #765. The pin
-remains explicit and immutable until the protected PR merges; it is not a
-moving `main` reference and it is not a LineageWeave monkey patch.
+commit `0246e91f107c8dbb82eb0b421d0aeb0f5fe21ecf`, the pushed integration
+head of ContextualWisdomLab/contextual-orchestrator#791. That PR combines
+the temperature negotiation in #779 with the orchestrator-owned embedding
+selection in #789 and remains stacked on #765. The pin remains explicit and
+immutable until the protected PRs merge; it is not a moving `main` reference
+and it is not a LineageWeave monkey patch.
 
 The runtime contract is:
 
@@ -33,6 +38,13 @@ The runtime contract is:
   reconciliation prompt; independent VISION worker evidence is retained instead.
 - A provider 4xx is reported as a failed orchestration attempt, never as a
   successful empty semantic result.
+- When HTTP 400/422 explicitly proves that `temperature` is unsupported, the
+  same endpoint is retried once with only that optional field omitted. Invalid
+  values and unrelated 4xx responses remain fail-closed.
+- Embedding requests may omit `model`; the orchestrator selects only an enabled
+  embedding-capable agent and returns the resolved model for provenance.
+- An omitted embedding model fails with `embedding_unavailable` when no eligible
+  agent exists; LineageWeave never supplies a heuristic or provider fallback.
 - An empty seed model is expanded from the configured gateway `/v1/models`
   endpoint; embedding-only rows are not added to the chat agent pool.
 - `json_object`, `json_schema`, and Responses JSON formats run conduct plus
@@ -44,4 +56,5 @@ The runtime contract is:
   implementation.
 - Rebuilding the image is required after the upstream pin changes.
 - Protected-branch review and merge remain external gates; this pin does not
-  bypass PR #765.
+  bypass ContextualWisdomLab/contextual-orchestrator#765 or
+  ContextualWisdomLab/contextual-orchestrator#779, #789, or #791.

docs/adr/0118-orchestrator-owned-embedding-model-provenance.md

@@ -0,0 +1,49 @@
+# ADR 0118: Orchestrator-owned embedding model discovery and provenance
+
+- Status: Accepted
+- Date: 2026-08-20
+- Scope: LineageWeave embedding ingestion and post-content backfill
+- Figma: N/A
+
+## Context
+
+LineageWeave must send embedding work through contextual-orchestrator. The
+orchestrator can select an embedding-capable agent when the request omits a
+model, but LineageWeave still read `LLM_GATEWAY_EMBEDDING_MODEL` and treated a
+local model string as a prerequisite. That made the consumer contradict the
+paper-grounded model boundary in ADR 0076 and the shared agent contract.
+
+## Decision
+
+1. LineageWeave does not read or expose a provider embedding model selector.
+2. The embedding request omits `model` when no explicit compatibility value is
+   supplied; contextual-orchestrator resolves the capability and returns the
+   served model code.
+3. Every persisted unit or visual-region embedding stores that returned model
+   code in the normalized embedding table. If the boundary returns no model
+   code, the vector is not persisted as provenance-bearing evidence and the
+   ingestion job remains incomplete for retry.
+4. Completeness checks in automatic mode require an embedding for every
+   embeddable unit and described image region, regardless of model code. An
+   explicit legacy model code, when supplied by a lower-level caller, retains
+   exact-model completeness matching but is not a runtime configuration path.
+5. The upstream contextual-orchestrator capability-selection contract is a
+   dependency of this consumer change. No monkey patch, provider SDK, local
+   model ranking, or sentinel model name is permitted.
+
+## Consequences
+
+- Runtime configuration contains only the internal orchestrator boundary; the
+  provider credentials remain in `~/.env` at the orchestrator service boundary.
+- Existing rows retain their actual `embedding_model_code`; no backfill uses a
+  guessed or synthetic model code.
+- A temporarily incomplete orchestrator response fails closed for evidence,
+  while the source body remains available for retry.
+
+## References
+
+ContextualWisdomLab. (2026). *contextual-orchestrator capability selection
+contract* (PR #789). https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/789
+
+ContextualWisdomLab. (2026). *ADR 0076: Paper-grounded model policy*.
+`docs/adr/0076-paper-grounded-model-policy.md`.

lineageweave/embedding_client.py

@@ -54,15 +54,19 @@ def embed(self, text: str) -> list[float]:
 
 
 class ContextualOrchestratorEmbeddingClient:
-    """Submit embeddings through contextual-orchestrator's batch boundary."""
+    """Submit embeddings through contextual-orchestrator's batch boundary.
+
+    Model discovery belongs to the orchestrator.  The returned model code is
+    retained for persistence so an automatic route still has provenance.
+    """
 
     available = True
 
     def __init__(
         self,
         base_url: str,
         api_key: str,
-        model: str,
+        model: str | None,
         *,
         timeout: float = 60.0,
         poll_interval: float = 0.25,
@@ -71,7 +75,8 @@ def __init__(
         if not self._base_url.endswith("/v1"):
             self._base_url = f"{self._base_url}/v1"
         self._api_key = api_key
-        self._model = model
+        self._model = model.strip() if isinstance(model, str) and model.strip() else None
+        self.resolved_model_code: str | None = self._model
         self._timeout = timeout
         self._poll_interval = poll_interval
 
@@ -84,17 +89,20 @@ def embed_many(self, texts: list[str]) -> list[list[float]]:
         if not texts:
             return []
         headers = {"authorization": f"Bearer {self._api_key}"}
+        payload: dict[str, object] = {
+            "inputs": texts,
+            "endpoint": "/v1/embeddings",
+            "metadata": {"service": "lineageweave", "channel": "post_content_embedding"},
+        }
+        if self._model:
+            payload["model"] = self._model
         response = post_json(
             f"{self._base_url}/batch/embeddings",
-            {
-                "model": self._model,
-                "inputs": texts,
-                "endpoint": "/v1/embeddings",
-                "metadata": {"service": "lineageweave", "channel": "post_content_embedding"},
-            },
+            payload,
             headers=headers,
             timeout=self._timeout,
         )
+        self._record_resolved_model(response)
         batch_id = response.get("batch_id")
         if isinstance(batch_id, str) and batch_id:
             deadline = time.monotonic() + self._timeout
@@ -112,6 +120,7 @@ def embed_many(self, texts: list[str]) -> list[list[float]]:
                     headers=headers,
                     timeout=self._timeout,
                 )
+                self._record_resolved_model(response)
 
         vectors = self._vectors(response, len(texts))
         if vectors is None:
@@ -139,10 +148,16 @@ def _vectors(response: dict, expected_count: int) -> list[list[float]] | None:
             return None
         return [vector for vector in ordered if vector is not None]
 
+    def _record_resolved_model(self, response: dict) -> None:
+        """Keep the model selected by the orchestrator for DB provenance."""
+        model = response.get("model")
+        if isinstance(model, str) and model.strip():
+            self.resolved_model_code = model.strip()
+
 
-def orchestrator_embedding_client(base_url: str, api_key: str, model: str):
+def orchestrator_embedding_client(base_url: str, api_key: str, model: str | None = None):
     """Build the batch embedding channel, or the unavailable null client."""
-    if not (base_url and api_key and model):
+    if not (base_url and api_key):
         return NullEmbeddingClient()
     return ContextualOrchestratorEmbeddingClient(base_url, api_key, model)