diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 743d2a7093..3e2aa06366 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -154,6 +154,8 @@ def FieldWithDefault(default_factory: Callable, **kwargs) -> Any: VALID_RECALL_FACT_TYPES, DryRunExtractionResult, MemoryFact, + MinScores, + RecallScores, TokenUsage, ) from hindsight_api.engine.search.tags import TagGroup, TagsMatch @@ -312,6 +314,15 @@ class RecallRequest(BaseModel): description="Compound tag filter using boolean groups. Groups in the list are AND-ed. " "Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}.", ) + min_scores: MinScores | None = Field( + default=None, + description="Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are " + "retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for " + "this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left " + "unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use " + "with care — the reranker's absolute scores are not calibrated across queries (a clearly-relevant match " + "may score ~0.001 even though it is ranked first).", + ) @field_validator("query") @classmethod @@ -367,6 +378,7 @@ class RecallResult(BaseModel): source_fact_ids: list[str] | None = ( None # IDs of source facts (observation type only, when source_facts is enabled) ) + scores: RecallScores | None = None # Per-stage recall scores (final/reranker/semantic/text) class EntityObservationResponse(BaseModel): @@ -3857,6 +3869,7 @@ async def api_recall( tags=request.tags, tags_match=request.tags_match, tag_groups=request.tag_groups, + min_scores=request.min_scores, ), operation="recall", bank_id=bank_id, @@ -3878,6 +3891,7 @@ def _fact_to_result(fact: "MemoryFact") -> RecallResult: chunk_id=fact.chunk_id, tags=fact.tags, source_fact_ids=fact.source_fact_ids, + scores=fact.scores, ) recall_results = [_fact_to_result(fact) for fact in core_result.results] diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index fdd69a6441..b36f1849a7 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -360,6 +360,8 @@ def validate_sql_schema(sql: str) -> None: EntityState, LLMCallTrace, MemoryFact, + MinScores, + RecallScores, ReflectResult, TokenUsage, ToolCallTrace, @@ -3919,6 +3921,7 @@ async def recall_async( tag_groups: list[TagGroup] | None = None, created_after: datetime | None = None, created_before: datetime | None = None, + min_scores: MinScores | None = None, _connection_budget: int | None = None, _quiet: bool = False, reranking: RecallReranking = "cross_encoder", @@ -4084,6 +4087,7 @@ async def recall_async( tag_groups=tag_groups, created_after=created_after, created_before=created_before, + min_scores=min_scores, connection_budget=_connection_budget, quiet=_quiet, include_source_facts=include_source_facts, @@ -4221,6 +4225,7 @@ async def _search_with_retries( tag_groups: list[TagGroup] | None = None, created_after: datetime | None = None, created_before: datetime | None = None, + min_scores: MinScores | None = None, connection_budget: int | None = None, quiet: bool = False, include_source_facts: bool = False, @@ -4357,6 +4362,8 @@ async def _search_with_retries( tag_groups=tag_groups, created_after=created_after, created_before=created_before, + min_semantic=min_scores.semantic if min_scores else None, + min_keyword=min_scores.keyword if min_scores else None, ) parallel_duration = time.time() - parallel_start finally: @@ -4723,6 +4730,30 @@ def to_tuple_format(results): if strategy_boosts: log_buffer.append(f" [4.7] Strategy boosts applied: {strategy_boosts}") + # Step 4.9: post-query min_scores filters (reranker + final). The + # semantic/text floors are applied earlier inside the SQL arms (see + # retrieve_semantic_bm25_combined); here we apply the post-rank floors on + # the scored results, after the final sort and before truncation, so every + # downstream step (prefer_observations dedup, truncation, token filtering) + # operates on the filtered set. Inclusive (>=), AND-ed, opt-in: a None + # threshold is a no-op. There is deliberately no default — the + # cross-encoder's absolute scores are not calibrated for a fixed cutoff + # (a clearly-relevant match can score ~0.001 while its *ranking* is right). + min_reranker = min_scores.reranker if min_scores else None + min_final = min_scores.final if min_scores else None + if (min_reranker is not None or min_final is not None) and scored_results: + before_min_score = len(scored_results) + scored_results = [ + sr + for sr in scored_results + if (min_reranker is None or sr.cross_encoder_score_normalized >= min_reranker) + and (min_final is None or sr.weight >= min_final) + ] + log_buffer.append( + f" [4.9] min_scores(reranker={min_reranker}, final={min_final}): " + f"{before_min_score}->{len(scored_results)} results" + ) + # Add reranked results to tracer AFTER combined scoring (so normalized values are included) if tracer: results_dict = [sr.to_dict() for sr in scored_results] @@ -5134,6 +5165,25 @@ def _make_source_fact(sid: str, r: Any) -> MemoryFact: ) # Convert results to MemoryFact objects + # Build per-result scores (final/reranker/semantic/text) keyed by id. + # reranker is None when the configured reranker is a passthrough (rrf / + # interleave modes, or the RRFPassthroughCrossEncoder), since its + # cross_encoder_score_normalized is then a rank-derived placeholder, not a + # true relevance score. + ce_model = self._cross_encoder_reranker.cross_encoder + reranker_passthrough = (reranking != "cross_encoder") or ( + ce_model is not None and getattr(ce_model, "provider_name", None) == "rrf" + ) + scores_by_id: dict[str, RecallScores] = { + sr.id: RecallScores( + final=sr.weight, + reranker=None if reranker_passthrough else sr.cross_encoder_score_normalized, + semantic=sr.candidate.arm_scores.semantic, + keyword=sr.candidate.arm_scores.keyword, + ) + for sr in top_scored + } + memory_facts = [] for result_dict in top_results_dicts: result_id = str(result_dict.get("id")) @@ -5157,6 +5207,7 @@ def _make_source_fact(sid: str, r: Any) -> MemoryFact: chunk_id=result_dict.get("chunk_id"), tags=result_dict.get("tags"), source_fact_ids=source_fact_ids_by_obs.get(result_id) if include_source_facts else None, + scores=scores_by_id.get(result_id), ) ) diff --git a/hindsight-api-slim/hindsight_api/engine/response_models.py b/hindsight-api-slim/hindsight_api/engine/response_models.py index e01862de2d..51ef5e366b 100644 --- a/hindsight-api-slim/hindsight_api/engine/response_models.py +++ b/hindsight-api-slim/hindsight_api/engine/response_models.py @@ -172,6 +172,47 @@ class DispositionTraits(BaseModel): model_config = ConfigDict(json_schema_extra={"example": {"skepticism": 3, "literalism": 3, "empathy": 3}}) +class RecallScores(BaseModel): + """Per-result recall scores from different stages of the pipeline. + + ``final`` is the value results are ranked by. The others are diagnostic and + can be filtered on via the recall ``min_scores`` request parameter. ``semantic`` + and ``keyword`` are the raw per-strategy retrieval scores (``None`` when that + strategy did not surface this result); ``reranker`` is the cross-encoder's + normalized relevance. + """ + + final: float = Field(description="Final ranking score (combined reranker + recency/temporal/proof boosts)") + reranker: float | None = Field( + default=None, + description="Cross-encoder relevance, normalized 0-1. None when the reranker is a passthrough (rrf/interleave modes).", + ) + semantic: float | None = Field( + default=None, description="Vector cosine similarity (0-1). None if this result was not surfaced semantically." + ) + keyword: float | None = Field( + default=None, + description="Keyword/full-text (BM25) score (>= 0, unbounded). None if this result was not surfaced by keyword search.", + ) + + +class MinScores(BaseModel): + """Optional per-stage score floors for recall (all inclusive, AND-ed). + + ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL + arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` + config for this request), so they prune weak matches before fusion. ``reranker`` + and ``final`` are **post-query** filters applied to the scored results after + reranking. Any field left None imposes no floor; all-None (the default) means + no score filtering. + """ + + semantic: float | None = Field(default=None, description="Retrieval-level: minimum vector similarity (0-1).") + keyword: float | None = Field(default=None, description="Retrieval-level: minimum keyword/full-text (BM25) score.") + reranker: float | None = Field(default=None, description="Post-query: minimum normalized reranker score (0-1).") + final: float | None = Field(default=None, description="Post-query: minimum final ranking score.") + + class MemoryFact(BaseModel): """ A single memory fact returned by search or think operations. @@ -231,6 +272,10 @@ def parse_metadata(cls, v: Any) -> dict[str, str] | None: None, description="IDs of source facts this observation was derived from (observation type only, when source_facts is enabled)", ) + scores: RecallScores | None = Field( + None, + description="Recall scores from each pipeline stage (final/reranker/semantic/text). Not returned for source facts.", + ) class ChunkInfo(BaseModel): diff --git a/hindsight-api-slim/hindsight_api/engine/search/fusion.py b/hindsight-api-slim/hindsight_api/engine/search/fusion.py index 4903e8ef70..4ef33493d5 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/fusion.py +++ b/hindsight-api-slim/hindsight_api/engine/search/fusion.py @@ -2,7 +2,7 @@ Helper functions for hybrid search (semantic + BM25 + graph). """ -from .types import MergedCandidate, RetrievalResult +from .types import ArmScores, MergedCandidate, RetrievalResult def cap_per_source(results: list[RetrievalResult], cap: int) -> list[RetrievalResult]: @@ -51,6 +51,7 @@ def reciprocal_rank_fusion(result_lists: list[list[RetrievalResult]], k: int = 6 rrf_scores = {} source_ranks = {} # Track rank from each source for each doc_id all_retrievals = {} # Store the actual RetrievalResult (use first occurrence) + arm_scores: dict[str, ArmScores] = {} # doc_id -> raw per-strategy scores across arms source_names = ["semantic", "bm25", "graph", "temporal"] @@ -79,17 +80,29 @@ def reciprocal_rank_fusion(result_lists: list[list[RetrievalResult]], k: int = 6 if doc_id not in rrf_scores: rrf_scores[doc_id] = 0.0 source_ranks[doc_id] = {} + arm_scores[doc_id] = ArmScores() rrf_scores[doc_id] += 1.0 / (k + rank) source_ranks[doc_id][f"{source_name}_rank"] = rank + # Capture this arm's raw score for the doc (the merged RetrievalResult + # below keeps only the first arm's score, so record each arm here). + if source_name == "semantic" and retrieval.similarity is not None: + arm_scores[doc_id].semantic = retrieval.similarity + elif source_name == "bm25" and retrieval.bm25_score is not None: + arm_scores[doc_id].keyword = retrieval.bm25_score + # Combine into final results with metadata merged_results = [] for rrf_rank, (doc_id, rrf_score) in enumerate( sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True), start=1 ): merged_candidate = MergedCandidate( - retrieval=all_retrievals[doc_id], rrf_score=rrf_score, rrf_rank=rrf_rank, source_ranks=source_ranks[doc_id] + retrieval=all_retrievals[doc_id], + rrf_score=rrf_score, + rrf_rank=rrf_rank, + source_ranks=source_ranks[doc_id], + arm_scores=arm_scores[doc_id], ) merged_results.append(merged_candidate) @@ -118,6 +131,7 @@ def interleave_fusion(result_lists: list[list[RetrievalResult]]) -> list[MergedC source_names = ["semantic", "bm25", "graph", "temporal"] source_ranks: dict[str, dict[str, int]] = {} all_retrievals: dict[str, RetrievalResult] = {} + arm_scores: dict[str, ArmScores] = {} for source_idx, results in enumerate(result_lists): source_name = source_names[source_idx] if source_idx < len(source_names) else f"source_{source_idx}" @@ -129,6 +143,11 @@ def interleave_fusion(result_lists: list[list[RetrievalResult]]) -> list[MergedC doc_id = retrieval.id all_retrievals.setdefault(doc_id, retrieval) source_ranks.setdefault(doc_id, {})[f"{source_name}_rank"] = rank + arm = arm_scores.setdefault(doc_id, ArmScores()) + if source_name == "semantic" and retrieval.similarity is not None: + arm.semantic = retrieval.similarity + elif source_name == "bm25" and retrieval.bm25_score is not None: + arm.keyword = retrieval.bm25_score # Round-robin pick across arms in priority order: all #1s, then all #2s, ... ordered_ids: list[str] = [] @@ -151,6 +170,7 @@ def interleave_fusion(result_lists: list[list[RetrievalResult]]) -> list[MergedC rrf_score=float(n - pos), rrf_rank=pos + 1, source_ranks=source_ranks[doc_id], + arm_scores=arm_scores[doc_id], ) for pos, doc_id in enumerate(ordered_ids) ] diff --git a/hindsight-api-slim/hindsight_api/engine/search/reranking.py b/hindsight-api-slim/hindsight_api/engine/search/reranking.py index 973b5f6f29..7f86323ece 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/reranking.py +++ b/hindsight-api-slim/hindsight_api/engine/search/reranking.py @@ -177,6 +177,9 @@ def apply_combined_scoring( else: # Neutral baseline is precisely 0.5, ensuring neutral multiplier (1.0) proof_norm = 0.5 + # Surface the proof signal so the trace can show the proof_count_boost + # factor (otherwise the reranked breakdown can't reconcile CE × boosts). + sr.proof_norm = proof_norm # RRF: kept at 0.0 for trace continuity but excluded from scoring. # RRF is batch-relative (min-max normalised) and redundant after reranking. diff --git a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py index 9cf866e639..56363a04e7 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/retrieval.py @@ -104,6 +104,8 @@ async def retrieve_semantic_bm25_combined( tag_groups: list[TagGroup] | None = None, created_after: datetime | None = None, created_before: datetime | None = None, + min_semantic: float | None = None, + min_keyword: float | None = None, ) -> dict[str, tuple[list[RetrievalResult], list[RetrievalResult]]]: """ Combined semantic + BM25 retrieval for multiple fact types in a single query. @@ -143,6 +145,12 @@ async def retrieve_semantic_bm25_combined( config = get_config() tokens = tokenize_query(query_text) + # Per-request retrieval-level score floors (recall min_scores.semantic / .keyword) + # override the global config defaults for this query, pruning weak matches in + # the SQL arms before fusion. + sem_min = min_semantic if min_semantic is not None else config.semantic_min_similarity + bm25_min = min_keyword if min_keyword is not None else config.bm25_min_score + # Over-fetch for HNSW approximation; semantic results trimmed to limit in Python. hnsw_fetch = max(limit * 5, 100) @@ -203,7 +211,7 @@ async def retrieve_semantic_bm25_combined( embedding_param="$1", bank_id_param="$2", fetch_limit=hnsw_fetch, - min_similarity=config.semantic_min_similarity, + min_similarity=sem_min, tags_clause=tags_clause, groups_clause=groups_clause, extra_where=created_range_clause, @@ -229,7 +237,7 @@ async def retrieve_semantic_bm25_combined( arm_index=i, text_search_extension=text_ext, bm25_language=config.text_search_extension_native_language, - bm25_min_score=config.bm25_min_score, + bm25_min_score=bm25_min, extra_where=created_range_clause, ) ) @@ -277,7 +285,7 @@ async def retrieve_semantic_bm25_combined( embedding_param="$1", bank_id_param="$2", fetch_limit=hnsw_fetch, - min_similarity=config.semantic_min_similarity, + min_similarity=sem_min, tags_clause=fb_tags_clause, groups_clause=fb_groups_clause, extra_where=fb_created_clause, @@ -706,6 +714,8 @@ async def retrieve_all_fact_types_parallel( tag_groups: list[TagGroup] | None = None, created_after: datetime | None = None, created_before: datetime | None = None, + min_semantic: float | None = None, + min_keyword: float | None = None, ) -> MultiFactTypeRetrievalResult: """ Optimized retrieval for multiple fact types using batched queries. @@ -766,6 +776,8 @@ async def retrieve_all_fact_types_parallel( tag_groups=tag_groups, created_after=created_after, created_before=created_before, + min_semantic=min_semantic, + min_keyword=min_keyword, ) semantic_bm25_time = time.time() - semantic_bm25_start @@ -781,7 +793,7 @@ async def retrieve_all_fact_types_parallel( tc_start, tc_end, budget=thinking_budget, - semantic_threshold=0.1, + semantic_threshold=min_semantic if min_semantic is not None else 0.1, tags=tags, tags_match=tags_match, tag_groups=tag_groups, diff --git a/hindsight-api-slim/hindsight_api/engine/search/tracer.py b/hindsight-api-slim/hindsight_api/engine/search/tracer.py index 4cf3947219..2ee9e9db16 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/tracer.py +++ b/hindsight-api-slim/hindsight_api/engine/search/tracer.py @@ -392,7 +392,7 @@ def add_reranked(self, reranked_results: list[dict[str, Any]], rrf_merged: list) # Extract score components (only include non-None values) # Keys from ScoredResult.to_dict(): cross_encoder_score, cross_encoder_score_normalized, - # rrf_normalized, temporal, recency, combined_score, weight + # rrf_normalized, temporal, recency, proof_norm, combined_score, weight score_components = {} for key in [ "cross_encoder_score", @@ -401,6 +401,7 @@ def add_reranked(self, reranked_results: list[dict[str, Any]], rrf_merged: list) "rrf_normalized", "temporal", "recency", + "proof_norm", "combined_score", ]: if key in result and result[key] is not None: diff --git a/hindsight-api-slim/hindsight_api/engine/search/types.py b/hindsight-api-slim/hindsight_api/engine/search/types.py index 9b464fc901..61bbf7c5f3 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/types.py +++ b/hindsight-api-slim/hindsight_api/engine/search/types.py @@ -82,6 +82,20 @@ def from_db_row(cls, row: dict[str, Any]) -> "RetrievalResult": ) +@dataclass +class ArmScores: + """Raw per-strategy retrieval scores for a single doc, aggregated across arms. + + Fusion keeps only the first-seen RetrievalResult per doc, so its per-arm score + fields reflect just one arm. This captures each arm's raw score for the same doc + so the recall response can report them (and ``min_scores`` can filter on them). + ``None`` means the doc was not surfaced by that arm. + """ + + semantic: float | None = None # cosine similarity from the semantic arm + keyword: float | None = None # BM25 / full-text score from the keyword arm + + @dataclass class MergedCandidate: """ @@ -97,6 +111,7 @@ class MergedCandidate: rrf_score: float rrf_rank: int = 0 source_ranks: dict[str, int] = field(default_factory=dict) # method_name -> rank + arm_scores: "ArmScores" = field(default_factory=lambda: ArmScores()) # raw per-strategy scores @property def id(self) -> str: @@ -123,6 +138,7 @@ class ScoredResult: rrf_normalized: float = 0.0 recency: float = 0.5 temporal: float = 0.5 + proof_norm: float = 0.5 # log-normalized proof count (neutral 0.5); drives proof_count_boost # Final combined score combined_score: float = 0.0 @@ -179,6 +195,7 @@ def to_dict(self) -> dict[str, Any]: result["rrf_normalized"] = self.rrf_normalized result["temporal"] = self.temporal result["recency"] = self.recency + result["proof_norm"] = self.proof_norm result["combined_score"] = self.combined_score result["weight"] = self.weight result["activation"] = self.weight # Legacy field diff --git a/hindsight-api-slim/hindsight_api/mcp_tools.py b/hindsight-api-slim/hindsight_api/mcp_tools.py index 5ba52b0e0a..0f09fd9168 100644 --- a/hindsight-api-slim/hindsight_api/mcp_tools.py +++ b/hindsight-api-slim/hindsight_api/mcp_tools.py @@ -22,7 +22,7 @@ ) from hindsight_api.engine.audit import AuditEntry, AuditLogger from hindsight_api.engine.memory_engine import Budget -from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES +from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES, MinScores from hindsight_api.engine.search.tags import TagGroup from hindsight_api.extensions import OperationValidationError from hindsight_api.models import RequestContext @@ -838,6 +838,7 @@ async def recall( tags_match: str = "any", tag_groups: list[dict] | None = None, query_timestamp: str | None = None, + min_scores: dict | None = None, bank_id: str | None = None, ) -> str | dict: """ @@ -858,6 +859,11 @@ async def recall( Mutually exclusive with tags. query_timestamp: Temporal context for the query (ISO format, e.g., '2024-01-15T10:30:00Z'). Anchors relative temporal expressions and recency scoring. + min_scores: Optional per-stage score floors as an object with any of: "semantic", "keyword" + (retrieval-level cutoffs), "reranker", "final" (post-ranking). E.g. {"reranker": 0.5}. + All inclusive and AND-ed; omit for no score filtering. The reranker's absolute scores are + not calibrated across queries, so only threshold against scores you've calibrated for your + own data. bank_id: Optional bank to search in (defaults to session bank). Use for cross-bank operations. """ try: @@ -890,6 +896,8 @@ async def recall( recall_kwargs["tag_groups"] = _TAG_GROUP_LIST_ADAPTER.validate_python(tag_groups) if query_timestamp is not None: recall_kwargs["question_date"] = parse_timestamp(query_timestamp) + if min_scores is not None: + recall_kwargs["min_scores"] = MinScores.model_validate(min_scores) recall_result = await memory.recall_async(**recall_kwargs) @@ -916,6 +924,7 @@ async def recall( tags_match: str = "any", tag_groups: list[dict] | None = None, query_timestamp: str | None = None, + min_scores: dict | None = None, ) -> dict: """ Args: @@ -935,6 +944,11 @@ async def recall( Mutually exclusive with tags. query_timestamp: Temporal context for the query (ISO format, e.g., '2024-01-15T10:30:00Z'). Anchors relative temporal expressions and recency scoring. + min_scores: Optional per-stage score floors as an object with any of: "semantic", "keyword" + (retrieval-level cutoffs), "reranker", "final" (post-ranking). E.g. {"reranker": 0.5}. + All inclusive and AND-ed; omit for no score filtering. The reranker's absolute scores are + not calibrated across queries, so only threshold against scores you've calibrated for your + own data. """ try: target_bank = config.bank_id_resolver() @@ -966,6 +980,8 @@ async def recall( recall_kwargs["tag_groups"] = _TAG_GROUP_LIST_ADAPTER.validate_python(tag_groups) if query_timestamp is not None: recall_kwargs["question_date"] = parse_timestamp(query_timestamp) + if min_scores is not None: + recall_kwargs["min_scores"] = MinScores.model_validate(min_scores) recall_result = await memory.recall_async(**recall_kwargs) diff --git a/hindsight-api-slim/tests/test_recall_min_score.py b/hindsight-api-slim/tests/test_recall_min_score.py new file mode 100644 index 0000000000..79703f6e58 --- /dev/null +++ b/hindsight-api-slim/tests/test_recall_min_score.py @@ -0,0 +1,182 @@ +"""Tests for the recall `scores` object and the `min_scores` filters. + +Inserts memory_units with known content + real embeddings directly via SQL, then +verifies that recall_async: + - returns a `scores` object (final/reranker/semantic/text) on every result, + - applies the post-query floors (`reranker`, `final`) to the scored results, + - applies the retrieval-level floors (`semantic`, `text`) inside the SQL arms, + - is unchanged by the default (`min_scores=None`). + +Filtering is deterministic post/pre-processing, so these assertions are direct — +no LLM and no LLM-as-judge required (uses the mock provider). +""" + +import uuid + +import pytest +import pytest_asyncio + +from hindsight_api import MemoryEngine, RequestContext +from hindsight_api.engine.response_models import MinScores +from hindsight_api.engine.retain import embedding_utils + +# Shared hardcoded UUIDs (memory_units.id is a global PK) → serialize xdist workers +# onto one group to avoid pk conflicts, same as test_recall_time_range.py. +pytestmark = pytest.mark.xdist_group("recall_min_score") + +ID_A = "00000000-0000-0000-0000-0000000000a1" +ID_B = "00000000-0000-0000-0000-0000000000a2" +ID_C = "00000000-0000-0000-0000-0000000000a3" +ALL_IDS = (ID_A, ID_B, ID_C) + +RC = RequestContext(tenant_id="default") + + +async def _insert_fact(conn, *, fact_id: str, text: str, bank_id: str, embedding_str: str) -> None: + await conn.execute( + """ + INSERT INTO memory_units (id, bank_id, text, fact_type, embedding) + VALUES ($1, $2, $3, 'world', $4::vector) + """, + fact_id, + bank_id, + text, + embedding_str, + ) + + +@pytest_asyncio.fixture +async def seeded_memory(memory_no_llm_verify: MemoryEngine): + """Insert three facts with real embeddings and return (engine, bank_id).""" + engine = memory_no_llm_verify + bank_id = f"test-min-score-{uuid.uuid4().hex[:8]}" + + await engine.get_bank_profile(bank_id, request_context=RC) + + embeddings = await embedding_utils.generate_embeddings_batch( + engine.embeddings, + ["the cat sat on the mat", "dogs are loyal animals", "birds can fly in the sky"], + ) + + def _to_str(emb: list[float]) -> str: + return "[" + ",".join(str(v) for v in emb) + "]" + + pool = await engine._get_pool() + async with pool.acquire() as conn: + await conn.execute("DELETE FROM memory_units WHERE id IN ($1, $2, $3)", *ALL_IDS) + await _insert_fact( + conn, fact_id=ID_A, text="the cat sat on the mat", bank_id=bank_id, embedding_str=_to_str(embeddings[0]) + ) + await _insert_fact( + conn, fact_id=ID_B, text="dogs are loyal animals", bank_id=bank_id, embedding_str=_to_str(embeddings[1]) + ) + await _insert_fact( + conn, fact_id=ID_C, text="birds can fly in the sky", bank_id=bank_id, embedding_str=_to_str(embeddings[2]) + ) + + yield engine, bank_id + + await engine.delete_bank(bank_id, request_context=RC) + + +def _ids(result) -> set[str]: + return {str(r.id) for r in result.results} + + +async def _recall(engine, bank_id, *, query="animals and nature", **kwargs): + return await engine.recall_async( + bank_id=bank_id, + query=query, + request_context=RC, + max_tokens=10000, + **kwargs, + ) + + +class TestRecallScores: + async def test_every_result_has_scores(self, seeded_memory): + engine, bank_id = seeded_memory + result = await _recall(engine, bank_id) + assert result.results, "expected the seeded facts to be recalled" + for r in result.results: + assert r.scores is not None, f"result {r.id} is missing scores" + assert isinstance(r.scores.final, float) + # semantic surfaced these (vector arm) — should be populated and 0..1 + assert r.scores.semantic is not None + assert 0.0 <= r.scores.semantic <= 1.0 + + async def test_results_ordered_by_descending_final(self, seeded_memory): + engine, bank_id = seeded_memory + result = await _recall(engine, bank_id) + finals = [r.scores.final for r in result.results] + assert finals == sorted(finals, reverse=True), f"results not ordered by final score: {finals}" + + +class TestPostQueryFilters: + async def test_none_is_no_op(self, seeded_memory): + engine, bank_id = seeded_memory + baseline = await _recall(engine, bank_id) + explicit = await _recall(engine, bank_id, min_scores=None) + assert _ids(baseline) == _ids(explicit) + + async def test_final_floor_filters_and_is_a_subset(self, seeded_memory): + engine, bank_id = seeded_memory + baseline = await _recall(engine, bank_id) + finals = sorted((r.scores.final for r in baseline.results), reverse=True) + assert len(finals) >= 2, "need at least two results to exercise a mid threshold" + threshold = finals[-1] + (finals[-2] - finals[-1]) / 2 + + filtered = await _recall(engine, bank_id, min_scores=MinScores(final=threshold)) + assert _ids(filtered), "threshold should still keep the top result(s)" + assert _ids(filtered) < _ids(baseline), "threshold must drop at least one result" + for r in filtered.results: + assert r.scores.final >= threshold + + async def test_final_floor_above_all_returns_empty(self, seeded_memory): + engine, bank_id = seeded_memory + baseline = await _recall(engine, bank_id) + max_final = max(r.scores.final for r in baseline.results) + filtered = await _recall(engine, bank_id, min_scores=MinScores(final=max_final + 1.0)) + assert filtered.results == [], f"expected nothing above all final scores, got {_ids(filtered)}" + + async def test_reranker_floor_filters(self, seeded_memory): + engine, bank_id = seeded_memory + baseline = await _recall(engine, bank_id) + rerankers = sorted((r.scores.reranker for r in baseline.results if r.scores.reranker is not None)) + if len(rerankers) < 2: + pytest.skip("reranker scores unavailable (passthrough reranker)") + threshold = rerankers[-1] # keep only the top reranker score(s) + filtered = await _recall(engine, bank_id, min_scores=MinScores(reranker=threshold)) + assert len(filtered.results) < len(baseline.results) + for r in filtered.results: + assert r.scores.reranker is not None and r.scores.reranker >= threshold + + +class TestRetrievalLevelFilters: + async def test_semantic_floor_prunes_in_retrieval(self, seeded_memory): + """min_scores.semantic is a SQL-arm cutoff: every returned result has a + semantic score >= the floor, and a high floor returns nothing.""" + engine, bank_id = seeded_memory + baseline = await _recall(engine, bank_id) + sems = sorted(r.scores.semantic for r in baseline.results if r.scores.semantic is not None) + assert sems, "semantic arm should have surfaced these facts" + # A floor just above the lowest semantic score must drop that weakest result. + floor = sems[-1] + filtered = await _recall(engine, bank_id, min_scores=MinScores(semantic=floor)) + assert len(filtered.results) <= len(baseline.results) + for r in filtered.results: + assert r.scores.semantic is not None and r.scores.semantic >= floor + + async def test_semantic_floor_above_one_returns_empty(self, seeded_memory): + engine, bank_id = seeded_memory + filtered = await _recall(engine, bank_id, min_scores=MinScores(semantic=1.1)) + assert filtered.results == [] + + +class TestRecallRequestDefault: + """min_scores is opt-in: the HTTP recall defaults to None (no filtering).""" + + def test_http_request_defaults_to_none(self): + from hindsight_api.api.http import RecallRequest + + assert RecallRequest(query="hi").min_scores is None diff --git a/hindsight-cli/.openapi-coverage.toml b/hindsight-cli/.openapi-coverage.toml index 63e9837db9..9837bd8807 100644 --- a/hindsight-cli/.openapi-coverage.toml +++ b/hindsight-cli/.openapi-coverage.toml @@ -117,6 +117,7 @@ http_config = "Advanced HTTP customisation (headers/method/timeout/params) is no types = "CLI exposes this as --fact-type (the schema property is named `types` but it holds fact types)." include = "Flattened into --include-chunks / --chunk-max-tokens (facts are always included)." tag_groups = "Complex nested tag filter not yet exposed in the CLI; use --tags / --tags-match for simple cases." +min_scores = "Complex nested per-stage score floors (semantic/keyword/reranker/final) not yet exposed in the CLI." [fields.reflect] include = "Flattened into --include-facts and related flags." diff --git a/hindsight-cli/src/commands/explore.rs b/hindsight-cli/src/commands/explore.rs index dca0e0249a..c0bda06de7 100644 --- a/hindsight-cli/src/commands/explore.rs +++ b/hindsight-cli/src/commands/explore.rs @@ -345,6 +345,7 @@ impl App { tags: None, tags_match: TagsMatch::Any, tag_groups: None, + min_scores: None, }; let result = client.recall(&bank_id, &request, false) diff --git a/hindsight-cli/src/commands/memory.rs b/hindsight-cli/src/commands/memory.rs index c9c6311027..0ed6830e94 100644 --- a/hindsight-cli/src/commands/memory.rs +++ b/hindsight-cli/src/commands/memory.rs @@ -316,6 +316,7 @@ pub fn recall( tags: if tags.is_empty() { None } else { Some(tags) }, tags_match: parse_tags_match(&tags_match), tag_groups: None, + min_scores: None, }; let response = client.recall(agent_id, &request, verbose); diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index 66bec049ff..022ad115a5 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -6878,6 +6878,30 @@ components: nullable: true type: integer title: MentalModelTrigger + MinScores: + description: |- + Optional per-stage score floors for recall (all inclusive, AND-ed). + + ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL + arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` + config for this request), so they prune weak matches before fusion. ``reranker`` + and ``final`` are **post-query** filters applied to the scored results after + reranking. Any field left None imposes no floor; all-None (the default) means + no score filtering. + properties: + semantic: + nullable: true + type: number + keyword: + nullable: true + type: number + reranker: + nullable: true + type: number + final: + nullable: true + type: number + title: MinScores ObservationScope: description: "A distinct observation scope: an exact tag set plus its observation\ \ count." @@ -7174,6 +7198,8 @@ components: $ref: '#/components/schemas/MentalModelTrigger_Input_tag_groups_inner' nullable: true type: array + min_scores: + $ref: '#/components/schemas/MinScores' required: - query title: RecallRequest @@ -7297,10 +7323,39 @@ components: type: string nullable: true type: array + scores: + $ref: '#/components/schemas/RecallScores' required: - id - text title: RecallResult + RecallScores: + description: |- + Per-result recall scores from different stages of the pipeline. + + ``final`` is the value results are ranked by. The others are diagnostic and + can be filtered on via the recall ``min_scores`` request parameter. ``semantic`` + and ``keyword`` are the raw per-strategy retrieval scores (``None`` when that + strategy did not surface this result); ``reranker`` is the cross-encoder's + normalized relevance. + properties: + final: + description: Final ranking score (combined reranker + recency/temporal/proof + boosts) + title: Final + type: number + reranker: + nullable: true + type: number + semantic: + nullable: true + type: number + keyword: + nullable: true + type: number + required: + - final + title: RecallScores RecoverConsolidationResponse: description: Response model for recovering failed consolidation. example: diff --git a/hindsight-clients/go/model_min_scores.go b/hindsight-clients/go/model_min_scores.go new file mode 100644 index 0000000000..f349a12ba7 --- /dev/null +++ b/hindsight-clients/go/model_min_scores.go @@ -0,0 +1,274 @@ +/* +Hindsight HTTP API + +HTTP API for Hindsight + +API version: 0.8.3 +*/ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package hindsight + +import ( + "encoding/json" +) + +// checks if the MinScores type satisfies the MappedNullable interface at compile time +var _ MappedNullable = &MinScores{} + +// MinScores Optional per-stage score floors for recall (all inclusive, AND-ed). ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` config for this request), so they prune weak matches before fusion. ``reranker`` and ``final`` are **post-query** filters applied to the scored results after reranking. Any field left None imposes no floor; all-None (the default) means no score filtering. +type MinScores struct { + Semantic NullableFloat32 `json:"semantic,omitempty"` + Keyword NullableFloat32 `json:"keyword,omitempty"` + Reranker NullableFloat32 `json:"reranker,omitempty"` + Final NullableFloat32 `json:"final,omitempty"` +} + +// NewMinScores instantiates a new MinScores object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewMinScores() *MinScores { + this := MinScores{} + return &this +} + +// NewMinScoresWithDefaults instantiates a new MinScores object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewMinScoresWithDefaults() *MinScores { + this := MinScores{} + return &this +} + +// GetSemantic returns the Semantic field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *MinScores) GetSemantic() float32 { + if o == nil || IsNil(o.Semantic.Get()) { + var ret float32 + return ret + } + return *o.Semantic.Get() +} + +// GetSemanticOk returns a tuple with the Semantic field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *MinScores) GetSemanticOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Semantic.Get(), o.Semantic.IsSet() +} + +// HasSemantic returns a boolean if a field has been set. +func (o *MinScores) HasSemantic() bool { + if o != nil && o.Semantic.IsSet() { + return true + } + + return false +} + +// SetSemantic gets a reference to the given NullableFloat32 and assigns it to the Semantic field. +func (o *MinScores) SetSemantic(v float32) { + o.Semantic.Set(&v) +} +// SetSemanticNil sets the value for Semantic to be an explicit nil +func (o *MinScores) SetSemanticNil() { + o.Semantic.Set(nil) +} + +// UnsetSemantic ensures that no value is present for Semantic, not even an explicit nil +func (o *MinScores) UnsetSemantic() { + o.Semantic.Unset() +} + +// GetKeyword returns the Keyword field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *MinScores) GetKeyword() float32 { + if o == nil || IsNil(o.Keyword.Get()) { + var ret float32 + return ret + } + return *o.Keyword.Get() +} + +// GetKeywordOk returns a tuple with the Keyword field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *MinScores) GetKeywordOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Keyword.Get(), o.Keyword.IsSet() +} + +// HasKeyword returns a boolean if a field has been set. +func (o *MinScores) HasKeyword() bool { + if o != nil && o.Keyword.IsSet() { + return true + } + + return false +} + +// SetKeyword gets a reference to the given NullableFloat32 and assigns it to the Keyword field. +func (o *MinScores) SetKeyword(v float32) { + o.Keyword.Set(&v) +} +// SetKeywordNil sets the value for Keyword to be an explicit nil +func (o *MinScores) SetKeywordNil() { + o.Keyword.Set(nil) +} + +// UnsetKeyword ensures that no value is present for Keyword, not even an explicit nil +func (o *MinScores) UnsetKeyword() { + o.Keyword.Unset() +} + +// GetReranker returns the Reranker field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *MinScores) GetReranker() float32 { + if o == nil || IsNil(o.Reranker.Get()) { + var ret float32 + return ret + } + return *o.Reranker.Get() +} + +// GetRerankerOk returns a tuple with the Reranker field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *MinScores) GetRerankerOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Reranker.Get(), o.Reranker.IsSet() +} + +// HasReranker returns a boolean if a field has been set. +func (o *MinScores) HasReranker() bool { + if o != nil && o.Reranker.IsSet() { + return true + } + + return false +} + +// SetReranker gets a reference to the given NullableFloat32 and assigns it to the Reranker field. +func (o *MinScores) SetReranker(v float32) { + o.Reranker.Set(&v) +} +// SetRerankerNil sets the value for Reranker to be an explicit nil +func (o *MinScores) SetRerankerNil() { + o.Reranker.Set(nil) +} + +// UnsetReranker ensures that no value is present for Reranker, not even an explicit nil +func (o *MinScores) UnsetReranker() { + o.Reranker.Unset() +} + +// GetFinal returns the Final field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *MinScores) GetFinal() float32 { + if o == nil || IsNil(o.Final.Get()) { + var ret float32 + return ret + } + return *o.Final.Get() +} + +// GetFinalOk returns a tuple with the Final field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *MinScores) GetFinalOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Final.Get(), o.Final.IsSet() +} + +// HasFinal returns a boolean if a field has been set. +func (o *MinScores) HasFinal() bool { + if o != nil && o.Final.IsSet() { + return true + } + + return false +} + +// SetFinal gets a reference to the given NullableFloat32 and assigns it to the Final field. +func (o *MinScores) SetFinal(v float32) { + o.Final.Set(&v) +} +// SetFinalNil sets the value for Final to be an explicit nil +func (o *MinScores) SetFinalNil() { + o.Final.Set(nil) +} + +// UnsetFinal ensures that no value is present for Final, not even an explicit nil +func (o *MinScores) UnsetFinal() { + o.Final.Unset() +} + +func (o MinScores) MarshalJSON() ([]byte, error) { + toSerialize,err := o.ToMap() + if err != nil { + return []byte{}, err + } + return json.Marshal(toSerialize) +} + +func (o MinScores) ToMap() (map[string]interface{}, error) { + toSerialize := map[string]interface{}{} + if o.Semantic.IsSet() { + toSerialize["semantic"] = o.Semantic.Get() + } + if o.Keyword.IsSet() { + toSerialize["keyword"] = o.Keyword.Get() + } + if o.Reranker.IsSet() { + toSerialize["reranker"] = o.Reranker.Get() + } + if o.Final.IsSet() { + toSerialize["final"] = o.Final.Get() + } + return toSerialize, nil +} + +type NullableMinScores struct { + value *MinScores + isSet bool +} + +func (v NullableMinScores) Get() *MinScores { + return v.value +} + +func (v *NullableMinScores) Set(val *MinScores) { + v.value = val + v.isSet = true +} + +func (v NullableMinScores) IsSet() bool { + return v.isSet +} + +func (v *NullableMinScores) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableMinScores(val *MinScores) *NullableMinScores { + return &NullableMinScores{value: val, isSet: true} +} + +func (v NullableMinScores) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableMinScores) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + + diff --git a/hindsight-clients/go/model_recall_request.go b/hindsight-clients/go/model_recall_request.go index aafa70423c..a453d017d8 100644 --- a/hindsight-clients/go/model_recall_request.go +++ b/hindsight-clients/go/model_recall_request.go @@ -35,6 +35,7 @@ type RecallRequest struct { // How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged), 'exact' (set-equality on the full scope, excludes untagged). With 'exact' and no tags (or []), the empty global scope is selected and only untagged memories match. TagsMatch *string `json:"tags_match,omitempty"` TagGroups []MentalModelTriggerInputTagGroupsInner `json:"tag_groups,omitempty"` + MinScores NullableMinScores `json:"min_scores,omitempty"` } type _RecallRequest RecallRequest @@ -430,6 +431,48 @@ func (o *RecallRequest) SetTagGroups(v []MentalModelTriggerInputTagGroupsInner) o.TagGroups = v } +// GetMinScores returns the MinScores field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RecallRequest) GetMinScores() MinScores { + if o == nil || IsNil(o.MinScores.Get()) { + var ret MinScores + return ret + } + return *o.MinScores.Get() +} + +// GetMinScoresOk returns a tuple with the MinScores field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RecallRequest) GetMinScoresOk() (*MinScores, bool) { + if o == nil { + return nil, false + } + return o.MinScores.Get(), o.MinScores.IsSet() +} + +// HasMinScores returns a boolean if a field has been set. +func (o *RecallRequest) HasMinScores() bool { + if o != nil && o.MinScores.IsSet() { + return true + } + + return false +} + +// SetMinScores gets a reference to the given NullableMinScores and assigns it to the MinScores field. +func (o *RecallRequest) SetMinScores(v MinScores) { + o.MinScores.Set(&v) +} +// SetMinScoresNil sets the value for MinScores to be an explicit nil +func (o *RecallRequest) SetMinScoresNil() { + o.MinScores.Set(nil) +} + +// UnsetMinScores ensures that no value is present for MinScores, not even an explicit nil +func (o *RecallRequest) UnsetMinScores() { + o.MinScores.Unset() +} + func (o RecallRequest) MarshalJSON() ([]byte, error) { toSerialize,err := o.ToMap() if err != nil { @@ -471,6 +514,9 @@ func (o RecallRequest) ToMap() (map[string]interface{}, error) { if o.TagGroups != nil { toSerialize["tag_groups"] = o.TagGroups } + if o.MinScores.IsSet() { + toSerialize["min_scores"] = o.MinScores.Get() + } return toSerialize, nil } diff --git a/hindsight-clients/go/model_recall_result.go b/hindsight-clients/go/model_recall_result.go index a4e5061595..5849c9a794 100644 --- a/hindsight-clients/go/model_recall_result.go +++ b/hindsight-clients/go/model_recall_result.go @@ -34,6 +34,7 @@ type RecallResult struct { ChunkId NullableString `json:"chunk_id,omitempty"` Tags []string `json:"tags,omitempty"` SourceFactIds []string `json:"source_fact_ids,omitempty"` + Scores NullableRecallScores `json:"scores,omitempty"` } type _RecallResult RecallResult @@ -531,6 +532,48 @@ func (o *RecallResult) SetSourceFactIds(v []string) { o.SourceFactIds = v } +// GetScores returns the Scores field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RecallResult) GetScores() RecallScores { + if o == nil || IsNil(o.Scores.Get()) { + var ret RecallScores + return ret + } + return *o.Scores.Get() +} + +// GetScoresOk returns a tuple with the Scores field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RecallResult) GetScoresOk() (*RecallScores, bool) { + if o == nil { + return nil, false + } + return o.Scores.Get(), o.Scores.IsSet() +} + +// HasScores returns a boolean if a field has been set. +func (o *RecallResult) HasScores() bool { + if o != nil && o.Scores.IsSet() { + return true + } + + return false +} + +// SetScores gets a reference to the given NullableRecallScores and assigns it to the Scores field. +func (o *RecallResult) SetScores(v RecallScores) { + o.Scores.Set(&v) +} +// SetScoresNil sets the value for Scores to be an explicit nil +func (o *RecallResult) SetScoresNil() { + o.Scores.Set(nil) +} + +// UnsetScores ensures that no value is present for Scores, not even an explicit nil +func (o *RecallResult) UnsetScores() { + o.Scores.Unset() +} + func (o RecallResult) MarshalJSON() ([]byte, error) { toSerialize,err := o.ToMap() if err != nil { @@ -576,6 +619,9 @@ func (o RecallResult) ToMap() (map[string]interface{}, error) { if o.SourceFactIds != nil { toSerialize["source_fact_ids"] = o.SourceFactIds } + if o.Scores.IsSet() { + toSerialize["scores"] = o.Scores.Get() + } return toSerialize, nil } diff --git a/hindsight-clients/go/model_recall_scores.go b/hindsight-clients/go/model_recall_scores.go new file mode 100644 index 0000000000..09e021d8dd --- /dev/null +++ b/hindsight-clients/go/model_recall_scores.go @@ -0,0 +1,297 @@ +/* +Hindsight HTTP API + +HTTP API for Hindsight + +API version: 0.8.3 +*/ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package hindsight + +import ( + "encoding/json" + "bytes" + "fmt" +) + +// checks if the RecallScores type satisfies the MappedNullable interface at compile time +var _ MappedNullable = &RecallScores{} + +// RecallScores Per-result recall scores from different stages of the pipeline. ``final`` is the value results are ranked by. The others are diagnostic and can be filtered on via the recall ``min_scores`` request parameter. ``semantic`` and ``keyword`` are the raw per-strategy retrieval scores (``None`` when that strategy did not surface this result); ``reranker`` is the cross-encoder's normalized relevance. +type RecallScores struct { + // Final ranking score (combined reranker + recency/temporal/proof boosts) + Final float32 `json:"final"` + Reranker NullableFloat32 `json:"reranker,omitempty"` + Semantic NullableFloat32 `json:"semantic,omitempty"` + Keyword NullableFloat32 `json:"keyword,omitempty"` +} + +type _RecallScores RecallScores + +// NewRecallScores instantiates a new RecallScores object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewRecallScores(final float32) *RecallScores { + this := RecallScores{} + this.Final = final + return &this +} + +// NewRecallScoresWithDefaults instantiates a new RecallScores object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewRecallScoresWithDefaults() *RecallScores { + this := RecallScores{} + return &this +} + +// GetFinal returns the Final field value +func (o *RecallScores) GetFinal() float32 { + if o == nil { + var ret float32 + return ret + } + + return o.Final +} + +// GetFinalOk returns a tuple with the Final field value +// and a boolean to check if the value has been set. +func (o *RecallScores) GetFinalOk() (*float32, bool) { + if o == nil { + return nil, false + } + return &o.Final, true +} + +// SetFinal sets field value +func (o *RecallScores) SetFinal(v float32) { + o.Final = v +} + +// GetReranker returns the Reranker field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RecallScores) GetReranker() float32 { + if o == nil || IsNil(o.Reranker.Get()) { + var ret float32 + return ret + } + return *o.Reranker.Get() +} + +// GetRerankerOk returns a tuple with the Reranker field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RecallScores) GetRerankerOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Reranker.Get(), o.Reranker.IsSet() +} + +// HasReranker returns a boolean if a field has been set. +func (o *RecallScores) HasReranker() bool { + if o != nil && o.Reranker.IsSet() { + return true + } + + return false +} + +// SetReranker gets a reference to the given NullableFloat32 and assigns it to the Reranker field. +func (o *RecallScores) SetReranker(v float32) { + o.Reranker.Set(&v) +} +// SetRerankerNil sets the value for Reranker to be an explicit nil +func (o *RecallScores) SetRerankerNil() { + o.Reranker.Set(nil) +} + +// UnsetReranker ensures that no value is present for Reranker, not even an explicit nil +func (o *RecallScores) UnsetReranker() { + o.Reranker.Unset() +} + +// GetSemantic returns the Semantic field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RecallScores) GetSemantic() float32 { + if o == nil || IsNil(o.Semantic.Get()) { + var ret float32 + return ret + } + return *o.Semantic.Get() +} + +// GetSemanticOk returns a tuple with the Semantic field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RecallScores) GetSemanticOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Semantic.Get(), o.Semantic.IsSet() +} + +// HasSemantic returns a boolean if a field has been set. +func (o *RecallScores) HasSemantic() bool { + if o != nil && o.Semantic.IsSet() { + return true + } + + return false +} + +// SetSemantic gets a reference to the given NullableFloat32 and assigns it to the Semantic field. +func (o *RecallScores) SetSemantic(v float32) { + o.Semantic.Set(&v) +} +// SetSemanticNil sets the value for Semantic to be an explicit nil +func (o *RecallScores) SetSemanticNil() { + o.Semantic.Set(nil) +} + +// UnsetSemantic ensures that no value is present for Semantic, not even an explicit nil +func (o *RecallScores) UnsetSemantic() { + o.Semantic.Unset() +} + +// GetKeyword returns the Keyword field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RecallScores) GetKeyword() float32 { + if o == nil || IsNil(o.Keyword.Get()) { + var ret float32 + return ret + } + return *o.Keyword.Get() +} + +// GetKeywordOk returns a tuple with the Keyword field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RecallScores) GetKeywordOk() (*float32, bool) { + if o == nil { + return nil, false + } + return o.Keyword.Get(), o.Keyword.IsSet() +} + +// HasKeyword returns a boolean if a field has been set. +func (o *RecallScores) HasKeyword() bool { + if o != nil && o.Keyword.IsSet() { + return true + } + + return false +} + +// SetKeyword gets a reference to the given NullableFloat32 and assigns it to the Keyword field. +func (o *RecallScores) SetKeyword(v float32) { + o.Keyword.Set(&v) +} +// SetKeywordNil sets the value for Keyword to be an explicit nil +func (o *RecallScores) SetKeywordNil() { + o.Keyword.Set(nil) +} + +// UnsetKeyword ensures that no value is present for Keyword, not even an explicit nil +func (o *RecallScores) UnsetKeyword() { + o.Keyword.Unset() +} + +func (o RecallScores) MarshalJSON() ([]byte, error) { + toSerialize,err := o.ToMap() + if err != nil { + return []byte{}, err + } + return json.Marshal(toSerialize) +} + +func (o RecallScores) ToMap() (map[string]interface{}, error) { + toSerialize := map[string]interface{}{} + toSerialize["final"] = o.Final + if o.Reranker.IsSet() { + toSerialize["reranker"] = o.Reranker.Get() + } + if o.Semantic.IsSet() { + toSerialize["semantic"] = o.Semantic.Get() + } + if o.Keyword.IsSet() { + toSerialize["keyword"] = o.Keyword.Get() + } + return toSerialize, nil +} + +func (o *RecallScores) UnmarshalJSON(data []byte) (err error) { + // This validates that all required properties are included in the JSON object + // by unmarshalling the object into a generic map with string keys and checking + // that every required field exists as a key in the generic map. + requiredProperties := []string{ + "final", + } + + allProperties := make(map[string]interface{}) + + err = json.Unmarshal(data, &allProperties) + + if err != nil { + return err; + } + + for _, requiredProperty := range(requiredProperties) { + if _, exists := allProperties[requiredProperty]; !exists { + return fmt.Errorf("no value given for required property %v", requiredProperty) + } + } + + varRecallScores := _RecallScores{} + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + err = decoder.Decode(&varRecallScores) + + if err != nil { + return err + } + + *o = RecallScores(varRecallScores) + + return err +} + +type NullableRecallScores struct { + value *RecallScores + isSet bool +} + +func (v NullableRecallScores) Get() *RecallScores { + return v.value +} + +func (v *NullableRecallScores) Set(val *RecallScores) { + v.value = val + v.isSet = true +} + +func (v NullableRecallScores) IsSet() bool { + return v.isSet +} + +func (v *NullableRecallScores) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableRecallScores(val *RecallScores) *NullableRecallScores { + return &NullableRecallScores{value: val, isSet: true} +} + +func (v NullableRecallScores) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableRecallScores) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} + + diff --git a/hindsight-clients/python/.openapi-generator/FILES b/hindsight-clients/python/.openapi-generator/FILES index 250ac50037..83b5ba67a0 100644 --- a/hindsight-clients/python/.openapi-generator/FILES +++ b/hindsight-clients/python/.openapi-generator/FILES @@ -95,6 +95,7 @@ hindsight_client_api/models/mental_model_trigger_input.py hindsight_client_api/models/mental_model_trigger_input_tag_groups_inner.py hindsight_client_api/models/mental_model_trigger_output.py hindsight_client_api/models/mental_model_trigger_output_tag_groups_inner.py +hindsight_client_api/models/min_scores.py hindsight_client_api/models/model_not.py hindsight_client_api/models/not1.py hindsight_client_api/models/observation_scope.py @@ -107,6 +108,7 @@ hindsight_client_api/models/operations_list_response.py hindsight_client_api/models/recall_request.py hindsight_client_api/models/recall_response.py hindsight_client_api/models/recall_result.py +hindsight_client_api/models/recall_scores.py hindsight_client_api/models/recover_consolidation_response.py hindsight_client_api/models/reflect_based_on.py hindsight_client_api/models/reflect_directive.py diff --git a/hindsight-clients/python/hindsight_client_api/__init__.py b/hindsight-clients/python/hindsight_client_api/__init__.py index d1b40ea502..b0209c9deb 100644 --- a/hindsight-clients/python/hindsight_client_api/__init__.py +++ b/hindsight-clients/python/hindsight_client_api/__init__.py @@ -119,6 +119,7 @@ from hindsight_client_api.models.mental_model_trigger_input_tag_groups_inner import MentalModelTriggerInputTagGroupsInner from hindsight_client_api.models.mental_model_trigger_output import MentalModelTriggerOutput from hindsight_client_api.models.mental_model_trigger_output_tag_groups_inner import MentalModelTriggerOutputTagGroupsInner +from hindsight_client_api.models.min_scores import MinScores from hindsight_client_api.models.model_not import ModelNot from hindsight_client_api.models.not1 import Not1 from hindsight_client_api.models.observation_scope import ObservationScope @@ -131,6 +132,7 @@ from hindsight_client_api.models.recall_request import RecallRequest from hindsight_client_api.models.recall_response import RecallResponse from hindsight_client_api.models.recall_result import RecallResult +from hindsight_client_api.models.recall_scores import RecallScores from hindsight_client_api.models.recover_consolidation_response import RecoverConsolidationResponse from hindsight_client_api.models.reflect_based_on import ReflectBasedOn from hindsight_client_api.models.reflect_directive import ReflectDirective diff --git a/hindsight-clients/python/hindsight_client_api/models/__init__.py b/hindsight-clients/python/hindsight_client_api/models/__init__.py index 93d5fccc89..553689bdb5 100644 --- a/hindsight-clients/python/hindsight_client_api/models/__init__.py +++ b/hindsight-clients/python/hindsight_client_api/models/__init__.py @@ -89,6 +89,7 @@ from hindsight_client_api.models.mental_model_trigger_input_tag_groups_inner import MentalModelTriggerInputTagGroupsInner from hindsight_client_api.models.mental_model_trigger_output import MentalModelTriggerOutput from hindsight_client_api.models.mental_model_trigger_output_tag_groups_inner import MentalModelTriggerOutputTagGroupsInner +from hindsight_client_api.models.min_scores import MinScores from hindsight_client_api.models.model_not import ModelNot from hindsight_client_api.models.not1 import Not1 from hindsight_client_api.models.observation_scope import ObservationScope @@ -101,6 +102,7 @@ from hindsight_client_api.models.recall_request import RecallRequest from hindsight_client_api.models.recall_response import RecallResponse from hindsight_client_api.models.recall_result import RecallResult +from hindsight_client_api.models.recall_scores import RecallScores from hindsight_client_api.models.recover_consolidation_response import RecoverConsolidationResponse from hindsight_client_api.models.reflect_based_on import ReflectBasedOn from hindsight_client_api.models.reflect_directive import ReflectDirective diff --git a/hindsight-clients/python/hindsight_client_api/models/min_scores.py b/hindsight-clients/python/hindsight_client_api/models/min_scores.py new file mode 100644 index 0000000000..0a3bbcb6dd --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/min_scores.py @@ -0,0 +1,113 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.8.3 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt +from typing import Any, ClassVar, Dict, List, Optional, Union +from typing import Optional, Set +from typing_extensions import Self + +class MinScores(BaseModel): + """ + Optional per-stage score floors for recall (all inclusive, AND-ed). ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` config for this request), so they prune weak matches before fusion. ``reranker`` and ``final`` are **post-query** filters applied to the scored results after reranking. Any field left None imposes no floor; all-None (the default) means no score filtering. + """ # noqa: E501 + semantic: Optional[Union[StrictFloat, StrictInt]] = None + keyword: Optional[Union[StrictFloat, StrictInt]] = None + reranker: Optional[Union[StrictFloat, StrictInt]] = None + final: Optional[Union[StrictFloat, StrictInt]] = None + __properties: ClassVar[List[str]] = ["semantic", "keyword", "reranker", "final"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of MinScores from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # set to None if semantic (nullable) is None + # and model_fields_set contains the field + if self.semantic is None and "semantic" in self.model_fields_set: + _dict['semantic'] = None + + # set to None if keyword (nullable) is None + # and model_fields_set contains the field + if self.keyword is None and "keyword" in self.model_fields_set: + _dict['keyword'] = None + + # set to None if reranker (nullable) is None + # and model_fields_set contains the field + if self.reranker is None and "reranker" in self.model_fields_set: + _dict['reranker'] = None + + # set to None if final (nullable) is None + # and model_fields_set contains the field + if self.final is None and "final" in self.model_fields_set: + _dict['final'] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of MinScores from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "semantic": obj.get("semantic"), + "keyword": obj.get("keyword"), + "reranker": obj.get("reranker"), + "final": obj.get("final") + }) + return _obj + + diff --git a/hindsight-clients/python/hindsight_client_api/models/recall_request.py b/hindsight-clients/python/hindsight_client_api/models/recall_request.py index ef30f0ff28..896ec44b38 100644 --- a/hindsight-clients/python/hindsight_client_api/models/recall_request.py +++ b/hindsight-clients/python/hindsight_client_api/models/recall_request.py @@ -22,6 +22,7 @@ from hindsight_client_api.models.budget import Budget from hindsight_client_api.models.include_options import IncludeOptions from hindsight_client_api.models.mental_model_trigger_input_tag_groups_inner import MentalModelTriggerInputTagGroupsInner +from hindsight_client_api.models.min_scores import MinScores from typing import Optional, Set from typing_extensions import Self @@ -40,7 +41,8 @@ class RecallRequest(BaseModel): tags: Optional[List[StrictStr]] = None tags_match: Optional[StrictStr] = Field(default='any', description="How to match tags: 'any' (OR, includes untagged), 'all' (AND, includes untagged), 'any_strict' (OR, excludes untagged), 'all_strict' (AND, excludes untagged), 'exact' (set-equality on the full scope, excludes untagged). With 'exact' and no tags (or []), the empty global scope is selected and only untagged memories match.") tag_groups: Optional[List[MentalModelTriggerInputTagGroupsInner]] = None - __properties: ClassVar[List[str]] = ["query", "types", "prefer_observations", "budget", "max_tokens", "trace", "query_timestamp", "include", "tags", "tags_match", "tag_groups"] + min_scores: Optional[MinScores] = None + __properties: ClassVar[List[str]] = ["query", "types", "prefer_observations", "budget", "max_tokens", "trace", "query_timestamp", "include", "tags", "tags_match", "tag_groups", "min_scores"] @field_validator('tags_match') def tags_match_validate_enum(cls, value): @@ -101,6 +103,9 @@ def to_dict(self) -> Dict[str, Any]: if _item_tag_groups: _items.append(_item_tag_groups.to_dict()) _dict['tag_groups'] = _items + # override the default output from pydantic by calling `to_dict()` of min_scores + if self.min_scores: + _dict['min_scores'] = self.min_scores.to_dict() # set to None if types (nullable) is None # and model_fields_set contains the field if self.types is None and "types" in self.model_fields_set: @@ -121,6 +126,11 @@ def to_dict(self) -> Dict[str, Any]: if self.tag_groups is None and "tag_groups" in self.model_fields_set: _dict['tag_groups'] = None + # set to None if min_scores (nullable) is None + # and model_fields_set contains the field + if self.min_scores is None and "min_scores" in self.model_fields_set: + _dict['min_scores'] = None + return _dict @classmethod @@ -143,7 +153,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "include": IncludeOptions.from_dict(obj["include"]) if obj.get("include") is not None else None, "tags": obj.get("tags"), "tags_match": obj.get("tags_match") if obj.get("tags_match") is not None else 'any', - "tag_groups": [MentalModelTriggerInputTagGroupsInner.from_dict(_item) for _item in obj["tag_groups"]] if obj.get("tag_groups") is not None else None + "tag_groups": [MentalModelTriggerInputTagGroupsInner.from_dict(_item) for _item in obj["tag_groups"]] if obj.get("tag_groups") is not None else None, + "min_scores": MinScores.from_dict(obj["min_scores"]) if obj.get("min_scores") is not None else None }) return _obj diff --git a/hindsight-clients/python/hindsight_client_api/models/recall_result.py b/hindsight-clients/python/hindsight_client_api/models/recall_result.py index 88bf61d9f3..a02775fb57 100644 --- a/hindsight-clients/python/hindsight_client_api/models/recall_result.py +++ b/hindsight-clients/python/hindsight_client_api/models/recall_result.py @@ -19,6 +19,7 @@ from pydantic import BaseModel, ConfigDict, StrictStr from typing import Any, ClassVar, Dict, List, Optional +from hindsight_client_api.models.recall_scores import RecallScores from typing import Optional, Set from typing_extensions import Self @@ -39,7 +40,8 @@ class RecallResult(BaseModel): chunk_id: Optional[StrictStr] = None tags: Optional[List[StrictStr]] = None source_fact_ids: Optional[List[StrictStr]] = None - __properties: ClassVar[List[str]] = ["id", "text", "type", "entities", "context", "occurred_start", "occurred_end", "mentioned_at", "document_id", "metadata", "chunk_id", "tags", "source_fact_ids"] + scores: Optional[RecallScores] = None + __properties: ClassVar[List[str]] = ["id", "text", "type", "entities", "context", "occurred_start", "occurred_end", "mentioned_at", "document_id", "metadata", "chunk_id", "tags", "source_fact_ids", "scores"] model_config = ConfigDict( populate_by_name=True, @@ -80,6 +82,9 @@ def to_dict(self) -> Dict[str, Any]: exclude=excluded_fields, exclude_none=True, ) + # override the default output from pydantic by calling `to_dict()` of scores + if self.scores: + _dict['scores'] = self.scores.to_dict() # set to None if type (nullable) is None # and model_fields_set contains the field if self.type is None and "type" in self.model_fields_set: @@ -135,6 +140,11 @@ def to_dict(self) -> Dict[str, Any]: if self.source_fact_ids is None and "source_fact_ids" in self.model_fields_set: _dict['source_fact_ids'] = None + # set to None if scores (nullable) is None + # and model_fields_set contains the field + if self.scores is None and "scores" in self.model_fields_set: + _dict['scores'] = None + return _dict @classmethod @@ -159,7 +169,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "metadata": obj.get("metadata"), "chunk_id": obj.get("chunk_id"), "tags": obj.get("tags"), - "source_fact_ids": obj.get("source_fact_ids") + "source_fact_ids": obj.get("source_fact_ids"), + "scores": RecallScores.from_dict(obj["scores"]) if obj.get("scores") is not None else None }) return _obj diff --git a/hindsight-clients/python/hindsight_client_api/models/recall_scores.py b/hindsight-clients/python/hindsight_client_api/models/recall_scores.py new file mode 100644 index 0000000000..0bd42945a8 --- /dev/null +++ b/hindsight-clients/python/hindsight_client_api/models/recall_scores.py @@ -0,0 +1,108 @@ +# coding: utf-8 + +""" + Hindsight HTTP API + + HTTP API for Hindsight + + The version of the OpenAPI document: 0.8.3 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt +from typing import Any, ClassVar, Dict, List, Optional, Union +from typing import Optional, Set +from typing_extensions import Self + +class RecallScores(BaseModel): + """ + Per-result recall scores from different stages of the pipeline. ``final`` is the value results are ranked by. The others are diagnostic and can be filtered on via the recall ``min_scores`` request parameter. ``semantic`` and ``keyword`` are the raw per-strategy retrieval scores (``None`` when that strategy did not surface this result); ``reranker`` is the cross-encoder's normalized relevance. + """ # noqa: E501 + final: Union[StrictFloat, StrictInt] = Field(description="Final ranking score (combined reranker + recency/temporal/proof boosts)") + reranker: Optional[Union[StrictFloat, StrictInt]] = None + semantic: Optional[Union[StrictFloat, StrictInt]] = None + keyword: Optional[Union[StrictFloat, StrictInt]] = None + __properties: ClassVar[List[str]] = ["final", "reranker", "semantic", "keyword"] + + model_config = ConfigDict( + populate_by_name=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead + return json.dumps(self.to_dict()) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of RecallScores from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # set to None if reranker (nullable) is None + # and model_fields_set contains the field + if self.reranker is None and "reranker" in self.model_fields_set: + _dict['reranker'] = None + + # set to None if semantic (nullable) is None + # and model_fields_set contains the field + if self.semantic is None and "semantic" in self.model_fields_set: + _dict['semantic'] = None + + # set to None if keyword (nullable) is None + # and model_fields_set contains the field + if self.keyword is None and "keyword" in self.model_fields_set: + _dict['keyword'] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of RecallScores from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "final": obj.get("final"), + "reranker": obj.get("reranker"), + "semantic": obj.get("semantic"), + "keyword": obj.get("keyword") + }) + return _obj + + diff --git a/hindsight-clients/rust/src/lib.rs b/hindsight-clients/rust/src/lib.rs index 690aa16964..9cb2062d3d 100644 --- a/hindsight-clients/rust/src/lib.rs +++ b/hindsight-clients/rust/src/lib.rs @@ -148,6 +148,7 @@ mod tests { tags: None, tags_match: types::TagsMatch::Any, tag_groups: None, + min_scores: None, }; let recall_response = client .recall_memories(&bank_id, None, &recall_request) diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 65d3bb426a..f36ff4de0a 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -2641,6 +2641,45 @@ export type MentalModelTriggerOutput = { recall_chunks_max_tokens?: number | null; }; +/** + * MinScores + * + * Optional per-stage score floors for recall (all inclusive, AND-ed). + * + * ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL + * arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` + * config for this request), so they prune weak matches before fusion. ``reranker`` + * and ``final`` are **post-query** filters applied to the scored results after + * reranking. Any field left None imposes no floor; all-None (the default) means + * no score filtering. + */ +export type MinScores = { + /** + * Semantic + * + * Retrieval-level: minimum vector similarity (0-1). + */ + semantic?: number | null; + /** + * Keyword + * + * Retrieval-level: minimum keyword/full-text (BM25) score. + */ + keyword?: number | null; + /** + * Reranker + * + * Post-query: minimum normalized reranker score (0-1). + */ + reranker?: number | null; + /** + * Final + * + * Post-query: minimum final ranking score. + */ + final?: number | null; +}; + /** * ObservationScope * @@ -2940,6 +2979,10 @@ export type RecallRequest = { * Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}. */ tag_groups?: Array | null; + /** + * Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care — the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first). + */ + min_scores?: MinScores | null; }; /** @@ -3044,6 +3087,45 @@ export type RecallResult = { * Source Fact Ids */ source_fact_ids?: Array | null; + scores?: RecallScores | null; +}; + +/** + * RecallScores + * + * Per-result recall scores from different stages of the pipeline. + * + * ``final`` is the value results are ranked by. The others are diagnostic and + * can be filtered on via the recall ``min_scores`` request parameter. ``semantic`` + * and ``keyword`` are the raw per-strategy retrieval scores (``None`` when that + * strategy did not surface this result); ``reranker`` is the cross-encoder's + * normalized relevance. + */ +export type RecallScores = { + /** + * Final + * + * Final ranking score (combined reranker + recency/temporal/proof boosts) + */ + final: number; + /** + * Reranker + * + * Cross-encoder relevance, normalized 0-1. None when the reranker is a passthrough (rrf/interleave modes). + */ + reranker?: number | null; + /** + * Semantic + * + * Vector cosine similarity (0-1). None if this result was not surfaced semantically. + */ + semantic?: number | null; + /** + * Keyword + * + * Keyword/full-text (BM25) score (>= 0, unbounded). None if this result was not surfaced by keyword search. + */ + keyword?: number | null; }; /** diff --git a/hindsight-control-plane/src/app/api/recall/route.ts b/hindsight-control-plane/src/app/api/recall/route.ts index 9c5a5e2835..2908e7acb6 100644 --- a/hindsight-control-plane/src/app/api/recall/route.ts +++ b/hindsight-control-plane/src/app/api/recall/route.ts @@ -18,6 +18,7 @@ export async function POST(request: NextRequest) { query_timestamp, tags, tags_match, + min_scores, } = body; const response = await sdk.recallMemories({ @@ -34,6 +35,7 @@ export async function POST(request: NextRequest) { query_timestamp, tags, tags_match, + min_scores, }, }); diff --git a/hindsight-control-plane/src/components/search-debug-view.tsx b/hindsight-control-plane/src/components/search-debug-view.tsx index 05612b17e1..7affc6b02b 100644 --- a/hindsight-control-plane/src/components/search-debug-view.tsx +++ b/hindsight-control-plane/src/components/search-debug-view.tsx @@ -39,6 +39,12 @@ type Budget = "low" | "mid" | "high"; type TagsMatch = "any" | "all" | "any_strict" | "all_strict" | "exact"; type ViewMode = "results" | "trace" | "json"; +// Render a score at FULL precision — never round. Rounded scores hide meaningful +// differences (e.g. 0.001125 vs 0.001004 both render as "0.001"), which is exactly +// what makes the reranker's behaviour hard to read. `null`/`undefined` → em dash. +const fmtScore = (v: number | null | undefined): string => + v === null || v === undefined ? "—" : String(v); + export function SearchDebugView() { const t = useTranslations("searchDebug"); const { currentBank } = useBank(); @@ -364,7 +370,7 @@ export function SearchDebugView() { Observation {t("proofCount", { count: obs.proof_count || 1 })} - {t("relevance", { value: (obs.relevance || 0).toFixed(3) })} + {t("relevance", { value: fmtScore(obs.relevance) })} ))} @@ -384,7 +390,7 @@ export function SearchDebugView() { ) : ( results.map((result: any, idx: number) => { const visit = trace?.visits?.find((v: any) => v.node_id === result.id); - const score = visit ? visit.weights.final_weight : result.score || 0; + const score = visit ? visit.weights.final_weight : result.scores?.final; return ( )} + {result.scores && ( +
+ final {fmtScore(result.scores.final)} + {result.scores.reranker !== null && + result.scores.reranker !== undefined && ( + reranker {fmtScore(result.scores.reranker)} + )} + {result.scores.semantic !== null && + result.scores.semantic !== undefined && ( + semantic {fmtScore(result.scores.semantic)} + )} + {result.scores.keyword !== null && + result.scores.keyword !== undefined && ( + keyword {fmtScore(result.scores.keyword)} + )} +
+ )}
-
{(score ?? 0).toFixed(3)}
+
{fmtScore(score)}
{t("scoreLabel")}
@@ -653,11 +676,7 @@ export function SearchDebugView() {

- {( - r.score || - r.similarity || - 0 - ).toFixed(4)} + {fmtScore(r.score ?? r.similarity)}
@@ -794,7 +813,7 @@ export function SearchDebugView() { {r.text}

- {t("rrfScore")} {(r.rrf_score || r.score || 0).toFixed(4)} + {t("rrfScore")} {fmtScore(r.rrf_score ?? r.score)}
@@ -853,7 +872,8 @@ export function SearchDebugView() {
- ce × recency_boost(±10%) × temporal_boost(±10%) + reranker_score × recency_boost(±10%) × temporal_boost(±10%) × + proof_boost(±5%)
@@ -884,6 +904,21 @@ export function SearchDebugView() {
{displayResults.map((r: any, rIdx: number) => { const sc = r.score_components || {}; + // The combined score is CE × multiplicative boosts, where each + // boost = 1 + alpha·(signal − 0.5) (neutral 1.0 at signal 0.5). + // The trace carries the raw 0–1 signals, so derive the actual + // multipliers here — otherwise CE × "Rec 1.000" can't reproduce + // the displayed total (e.g. 0.999 × recency-boost 1.100 ≈ 1.099). + const boost = (signal: number, alpha: number) => + 1 + alpha * (signal - 0.5); + const recBoost = + sc.recency !== undefined ? boost(sc.recency, 0.2) : undefined; + const tmpBoost = + sc.temporal !== undefined ? boost(sc.temporal, 0.2) : undefined; + const proofBoost = + sc.proof_norm !== undefined + ? boost(sc.proof_norm, 0.1) + : undefined; return (
- = {(r.rerank_score || r.score || 0).toFixed(4)} + = {fmtScore(r.rerank_score ?? r.score)} {sc.cross_encoder_score_normalized !== undefined && ( - CE: {sc.cross_encoder_score_normalized.toFixed(3)} + reranker {fmtScore(sc.cross_encoder_score_normalized)} + + )} + {recBoost !== undefined && ( + + × rec {fmtScore(recBoost)} )} - {sc.temporal !== undefined && sc.temporal !== 0.5 && ( - - Tmp: {sc.temporal.toFixed(3)} + {tmpBoost !== undefined && sc.temporal !== 0.5 && ( + + × tmp {fmtScore(tmpBoost)} )} - {sc.recency !== undefined && ( - - Rec: {sc.recency.toFixed(3)} + {proofBoost !== undefined && sc.proof_norm !== 0.5 && ( + + × proof {fmtScore(proofBoost)} )}
diff --git a/hindsight-control-plane/src/lib/api.ts b/hindsight-control-plane/src/lib/api.ts index 4e4a3193bf..cfb383fd21 100644 --- a/hindsight-control-plane/src/lib/api.ts +++ b/hindsight-control-plane/src/lib/api.ts @@ -356,6 +356,12 @@ export class ControlPlaneClient { query_timestamp?: string; tags?: string[]; tags_match?: "any" | "all" | "any_strict" | "all_strict" | "exact"; + min_scores?: { + semantic?: number | null; + keyword?: number | null; + reranker?: number | null; + final?: number | null; + }; }) { return this.fetchApi("/api/recall", { method: "POST", diff --git a/hindsight-docs/docs/developer/api/recall.mdx b/hindsight-docs/docs/developer/api/recall.mdx index 28469ff994..af701dc554 100644 --- a/hindsight-docs/docs/developer/api/recall.mdx +++ b/hindsight-docs/docs/developer/api/recall.mdx @@ -358,6 +358,27 @@ With any other `tags_match` mode, absent or empty `tags` means "no tag filter" ( When set to `true`, the response includes a detailed debug trace covering the query embedding, entry points, per-strategy retrieval results, RRF fusion candidates, reranked results, temporal constraints detected, and per-phase timings. Has no effect on the retrieval logic itself. Useful for understanding why specific memories were or were not returned. +### min_scores + +An optional object of per-stage score floors, each compared **inclusively** (`>=`) against the matching field of a result's [`scores`](#scores) and AND-ed together. Any field you leave unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering at all. The four fields operate at **two different levels of the pipeline**: + +| field | level | effect | +|---|---|---| +| `semantic` | retrieval | minimum vector similarity, pushed into the SQL — prunes weak vector matches **before** fusion (overrides the global similarity minimum for this request) | +| `keyword` | retrieval | minimum keyword/full-text (BM25) score, pushed into the SQL — prunes weak keyword matches before fusion | +| `reranker` | post-query | minimum normalized cross-encoder score, applied to the ranked results | +| `final` | post-query | minimum final ranking score, applied to the ranked results | + +```json +{ "query": "...", "min_scores": { "reranker": 0.5 } } +``` + +The retrieval-level floors (`semantic`/`keyword`) change *which candidates are considered*, so they can also change the final ordering; the post-query floors (`reranker`/`final`) only drop already-ranked results. Because freed slots are **not** backfilled, any floor can return fewer results than the budget allows. + +**Use floors with care.** The reranker's scores are reliable for *ordering* but not as *absolute* values — a clearly-relevant memory can score `~0.001` on one query and `~1.0` on another, so a fixed cutoff risks silently dropping good results. Calibrate any threshold against the scores you actually observe (recall with no `min_scores` first and inspect the [`scores`](#scores) object). + +The threshold is compared against the same `score` value the response reports. See the note under [`score`](#score) on why the scale is relative, not absolute, before relying on a fixed threshold. + --- ## Response @@ -366,7 +387,7 @@ When set to `true`, the response includes a detailed debug trace covering the qu The main list of recalled facts, ordered by relevance. Relevance is computed by running four retrieval strategies in parallel — semantic similarity, BM25 keyword, graph traversal, and temporal — fusing their rankings with Reciprocal Rank Fusion (RRF), then re-scoring the merged candidates with a cross-encoder reranker against the original query. -Results do not include a numeric score. Raw retrieval scores are not meaningful on an absolute scale — a score of 0.8 from one query tells you nothing useful compared to a score of 0.8 from another. What matters is the relative ordering, which is already reflected in the list order. Agents should consume memories in order and let `max_tokens` determine how many fit, rather than filtering by score. +Each result carries a [`scores`](#scores) object (see below). Treat these as **relative** signals: they reflect the ranking within a single query, not an absolute, cross-query confidence — a `0.8` from one query is not comparable to a `0.8` from another. For most agents the right approach is to consume memories in order and let `max_tokens` determine how many fit, rather than filtering by score. The `scores` object (and the [`min_scores`](#min_scores) parameter) exist for callers that want to inspect the ranking or drop a low-confidence tail; calibrate any threshold against the scores you see on an unfiltered query. Each item in `results` has the following fields: @@ -418,6 +439,17 @@ The ID of the source text chunk this fact was extracted from. Used to cross-refe For `observation`-type results only: the IDs of the original facts this observation was synthesized from. Cross-references with `source_facts` in the response. `null` for other types or when `include.source_facts` is not enabled. +#### scores + +An object of the per-stage scores for this result. `null` for `source_facts` entries, which are attached by provenance rather than ranked. Fields: + +- **`final`** — the score this fact was ranked by (cross-encoder relevance × recency/temporal/evidence boosts). `results` is ordered by it descending. A relative signal, not a calibrated probability (see the note above). +- **`reranker`** — the cross-encoder's normalized relevance (`0`–`1`). `null` when the deployment uses a passthrough reranker (RRF/interleave modes). +- **`semantic`** — the raw vector cosine similarity (`0`–`1`). `null` if this result was not surfaced by semantic search. +- **`keyword`** — the raw keyword/full-text (BM25) score (`≥ 0`, unbounded). `null` if this result was not surfaced by keyword search. + +Each field is also a valid [`min_scores`](#min_scores) floor. + --- ### source_facts diff --git a/hindsight-docs/docs/developer/mcp-server.md b/hindsight-docs/docs/developer/mcp-server.md index d353ccf9f8..c421b88e0f 100644 --- a/hindsight-docs/docs/developer/mcp-server.md +++ b/hindsight-docs/docs/developer/mcp-server.md @@ -189,6 +189,7 @@ Search memories to provide personalized responses. | `tags` | list[string] | No | Filter memories by tags | | `tags_match` | string | No | Tag matching mode: `any` (default) or `all` | | `query_timestamp` | string | No | ISO 8601 timestamp — recall as if asking at this point in time; anchors relative temporal expressions and recency scoring | +| `min_scores` | object | No | Optional per-stage score floors, e.g. `{"reranker": 0.5}`. Keys: `semantic`/`keyword` (retrieval-level cutoffs), `reranker`/`final` (post-ranking). All inclusive and AND-ed; omit for no filtering. Reranker scores aren't calibrated across queries — calibrate before use | **Example:** ```json diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 1c352e9e0f..3b19a7e6ac 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -10332,6 +10332,61 @@ "title": "MentalModelTrigger", "description": "Trigger settings for a mental model." }, + "MinScores": { + "properties": { + "semantic": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Semantic", + "description": "Retrieval-level: minimum vector similarity (0-1)." + }, + "keyword": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Keyword", + "description": "Retrieval-level: minimum keyword/full-text (BM25) score." + }, + "reranker": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Reranker", + "description": "Post-query: minimum normalized reranker score (0-1)." + }, + "final": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Final", + "description": "Post-query: minimum final ranking score." + } + }, + "type": "object", + "title": "MinScores", + "description": "Optional per-stage score floors for recall (all inclusive, AND-ed).\n\n``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL\narms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score``\nconfig for this request), so they prune weak matches before fusion. ``reranker``\nand ``final`` are **post-query** filters applied to the scored results after\nreranking. Any field left None imposes no floor; all-None (the default) means\nno score filtering." + }, "ObservationScope": { "properties": { "tags": { @@ -10897,6 +10952,17 @@ ], "title": "Tag Groups", "description": "Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}." + }, + "min_scores": { + "anyOf": [ + { + "$ref": "#/components/schemas/MinScores" + }, + { + "type": "null" + } + ], + "description": "Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care \u2014 the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first)." } }, "type": "object", @@ -11183,6 +11249,16 @@ } ], "title": "Source Fact Ids" + }, + "scores": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecallScores" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -11215,6 +11291,57 @@ "type": "world" } }, + "RecallScores": { + "properties": { + "final": { + "type": "number", + "title": "Final", + "description": "Final ranking score (combined reranker + recency/temporal/proof boosts)" + }, + "reranker": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Reranker", + "description": "Cross-encoder relevance, normalized 0-1. None when the reranker is a passthrough (rrf/interleave modes)." + }, + "semantic": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Semantic", + "description": "Vector cosine similarity (0-1). None if this result was not surfaced semantically." + }, + "keyword": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Keyword", + "description": "Keyword/full-text (BM25) score (>= 0, unbounded). None if this result was not surfaced by keyword search." + } + }, + "type": "object", + "required": [ + "final" + ], + "title": "RecallScores", + "description": "Per-result recall scores from different stages of the pipeline.\n\n``final`` is the value results are ranked by. The others are diagnostic and\ncan be filtered on via the recall ``min_scores`` request parameter. ``semantic``\nand ``keyword`` are the raw per-strategy retrieval scores (``None`` when that\nstrategy did not surface this result); ``reranker`` is the cross-encoder's\nnormalized relevance." + }, "RecoverConsolidationResponse": { "properties": { "retried_count": { diff --git a/skills/hindsight-docs/references/developer/api/recall.md b/skills/hindsight-docs/references/developer/api/recall.md index 7a3a8a38db..019d6d4378 100644 --- a/skills/hindsight-docs/references/developer/api/recall.md +++ b/skills/hindsight-docs/references/developer/api/recall.md @@ -603,6 +603,27 @@ With any other `tags_match` mode, absent or empty `tags` means "no tag filter" ( When set to `true`, the response includes a detailed debug trace covering the query embedding, entry points, per-strategy retrieval results, RRF fusion candidates, reranked results, temporal constraints detected, and per-phase timings. Has no effect on the retrieval logic itself. Useful for understanding why specific memories were or were not returned. +### min_scores + +An optional object of per-stage score floors, each compared **inclusively** (`>=`) against the matching field of a result's [`scores`](#scores) and AND-ed together. Any field you leave unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering at all. The four fields operate at **two different levels of the pipeline**: + +| field | level | effect | +|---|---|---| +| `semantic` | retrieval | minimum vector similarity, pushed into the SQL — prunes weak vector matches **before** fusion (overrides the global similarity minimum for this request) | +| `keyword` | retrieval | minimum keyword/full-text (BM25) score, pushed into the SQL — prunes weak keyword matches before fusion | +| `reranker` | post-query | minimum normalized cross-encoder score, applied to the ranked results | +| `final` | post-query | minimum final ranking score, applied to the ranked results | + +```json +{ "query": "...", "min_scores": { "reranker": 0.5 } } +``` + +The retrieval-level floors (`semantic`/`keyword`) change *which candidates are considered*, so they can also change the final ordering; the post-query floors (`reranker`/`final`) only drop already-ranked results. Because freed slots are **not** backfilled, any floor can return fewer results than the budget allows. + +**Use floors with care.** The reranker's scores are reliable for *ordering* but not as *absolute* values — a clearly-relevant memory can score `~0.001` on one query and `~1.0` on another, so a fixed cutoff risks silently dropping good results. Calibrate any threshold against the scores you actually observe (recall with no `min_scores` first and inspect the [`scores`](#scores) object). + +The threshold is compared against the same `score` value the response reports. See the note under [`score`](#score) on why the scale is relative, not absolute, before relying on a fixed threshold. + --- ## Response @@ -611,7 +632,7 @@ When set to `true`, the response includes a detailed debug trace covering the qu The main list of recalled facts, ordered by relevance. Relevance is computed by running four retrieval strategies in parallel — semantic similarity, BM25 keyword, graph traversal, and temporal — fusing their rankings with Reciprocal Rank Fusion (RRF), then re-scoring the merged candidates with a cross-encoder reranker against the original query. -Results do not include a numeric score. Raw retrieval scores are not meaningful on an absolute scale — a score of 0.8 from one query tells you nothing useful compared to a score of 0.8 from another. What matters is the relative ordering, which is already reflected in the list order. Agents should consume memories in order and let `max_tokens` determine how many fit, rather than filtering by score. +Each result carries a [`scores`](#scores) object (see below). Treat these as **relative** signals: they reflect the ranking within a single query, not an absolute, cross-query confidence — a `0.8` from one query is not comparable to a `0.8` from another. For most agents the right approach is to consume memories in order and let `max_tokens` determine how many fit, rather than filtering by score. The `scores` object (and the [`min_scores`](#min_scores) parameter) exist for callers that want to inspect the ranking or drop a low-confidence tail; calibrate any threshold against the scores you see on an unfiltered query. Each item in `results` has the following fields: @@ -663,6 +684,17 @@ The ID of the source text chunk this fact was extracted from. Used to cross-refe For `observation`-type results only: the IDs of the original facts this observation was synthesized from. Cross-references with `source_facts` in the response. `null` for other types or when `include.source_facts` is not enabled. +#### scores + +An object of the per-stage scores for this result. `null` for `source_facts` entries, which are attached by provenance rather than ranked. Fields: + +- **`final`** — the score this fact was ranked by (cross-encoder relevance × recency/temporal/evidence boosts). `results` is ordered by it descending. A relative signal, not a calibrated probability (see the note above). +- **`reranker`** — the cross-encoder's normalized relevance (`0`–`1`). `null` when the deployment uses a passthrough reranker (RRF/interleave modes). +- **`semantic`** — the raw vector cosine similarity (`0`–`1`). `null` if this result was not surfaced by semantic search. +- **`keyword`** — the raw keyword/full-text (BM25) score (`≥ 0`, unbounded). `null` if this result was not surfaced by keyword search. + +Each field is also a valid [`min_scores`](#min_scores) floor. + --- ### source_facts diff --git a/skills/hindsight-docs/references/developer/mcp-server.md b/skills/hindsight-docs/references/developer/mcp-server.md index d353ccf9f8..c421b88e0f 100644 --- a/skills/hindsight-docs/references/developer/mcp-server.md +++ b/skills/hindsight-docs/references/developer/mcp-server.md @@ -189,6 +189,7 @@ Search memories to provide personalized responses. | `tags` | list[string] | No | Filter memories by tags | | `tags_match` | string | No | Tag matching mode: `any` (default) or `all` | | `query_timestamp` | string | No | ISO 8601 timestamp — recall as if asking at this point in time; anchors relative temporal expressions and recency scoring | +| `min_scores` | object | No | Optional per-stage score floors, e.g. `{"reranker": 0.5}`. Keys: `semantic`/`keyword` (retrieval-level cutoffs), `reranker`/`final` (post-ranking). All inclusive and AND-ed; omit for no filtering. Reranker scores aren't calibrated across queries — calibrate before use | **Example:** ```json diff --git a/skills/hindsight-docs/references/openapi.json b/skills/hindsight-docs/references/openapi.json index 1c352e9e0f..3b19a7e6ac 100644 --- a/skills/hindsight-docs/references/openapi.json +++ b/skills/hindsight-docs/references/openapi.json @@ -10332,6 +10332,61 @@ "title": "MentalModelTrigger", "description": "Trigger settings for a mental model." }, + "MinScores": { + "properties": { + "semantic": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Semantic", + "description": "Retrieval-level: minimum vector similarity (0-1)." + }, + "keyword": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Keyword", + "description": "Retrieval-level: minimum keyword/full-text (BM25) score." + }, + "reranker": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Reranker", + "description": "Post-query: minimum normalized reranker score (0-1)." + }, + "final": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Final", + "description": "Post-query: minimum final ranking score." + } + }, + "type": "object", + "title": "MinScores", + "description": "Optional per-stage score floors for recall (all inclusive, AND-ed).\n\n``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL\narms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score``\nconfig for this request), so they prune weak matches before fusion. ``reranker``\nand ``final`` are **post-query** filters applied to the scored results after\nreranking. Any field left None imposes no floor; all-None (the default) means\nno score filtering." + }, "ObservationScope": { "properties": { "tags": { @@ -10897,6 +10952,17 @@ ], "title": "Tag Groups", "description": "Compound tag filter using boolean groups. Groups in the list are AND-ed. Each group is a leaf {tags, match} or compound {and: [...]}, {or: [...]}, {not: ...}." + }, + "min_scores": { + "anyOf": [ + { + "$ref": "#/components/schemas/MinScores" + }, + { + "type": "null" + } + ], + "description": "Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care \u2014 the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first)." } }, "type": "object", @@ -11183,6 +11249,16 @@ } ], "title": "Source Fact Ids" + }, + "scores": { + "anyOf": [ + { + "$ref": "#/components/schemas/RecallScores" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -11215,6 +11291,57 @@ "type": "world" } }, + "RecallScores": { + "properties": { + "final": { + "type": "number", + "title": "Final", + "description": "Final ranking score (combined reranker + recency/temporal/proof boosts)" + }, + "reranker": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Reranker", + "description": "Cross-encoder relevance, normalized 0-1. None when the reranker is a passthrough (rrf/interleave modes)." + }, + "semantic": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Semantic", + "description": "Vector cosine similarity (0-1). None if this result was not surfaced semantically." + }, + "keyword": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Keyword", + "description": "Keyword/full-text (BM25) score (>= 0, unbounded). None if this result was not surfaced by keyword search." + } + }, + "type": "object", + "required": [ + "final" + ], + "title": "RecallScores", + "description": "Per-result recall scores from different stages of the pipeline.\n\n``final`` is the value results are ranked by. The others are diagnostic and\ncan be filtered on via the recall ``min_scores`` request parameter. ``semantic``\nand ``keyword`` are the raw per-strategy retrieval scores (``None`` when that\nstrategy did not surface this result); ``reranker`` is the cross-encoder's\nnormalized relevance." + }, "RecoverConsolidationResponse": { "properties": { "retried_count": {