Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions backend/app/global_ask_retrieval.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,12 @@ async def semantic_candidate_post_ids(

Project mentions, responsibility/affiliation evidence, Keyman names, and
organization/team catalogs use one ``ILIKE`` predicate per indexed column.
This preserves multilingual substring lookup without wrapping indexed fields
in an expression that forces a sequential scan. Ontology lookup codes are
applied only to graph lookup-code columns. The function never returns source
text and nomination never grants access.
Search-corroborated raw/canonical organization-name pairs additionally
nominate direct organization mentions and affiliated people's posts; pending
or uncorroborated aliases do not. This preserves multilingual substring lookup
without wrapping indexed fields in an expression that forces a sequential
scan. Ontology lookup codes are applied only to graph lookup-code columns.
The function never returns source text and nomination never grants access.
"""

if maximum_candidates <= 0:
Expand All @@ -84,6 +86,15 @@ async def semantic_candidate_post_ids(
"""
with query_terms as (
select unnest($1::text[]) as term
), verified_organization as (
select distinct entity.corporate_entity_id
from organization_name_resolution resolution
join corporate_entity entity
on entity.entity_name = resolution.resolved_organization_name
join query_terms term
on resolution.raw_organization_name ilike '%' || term.term || '%'
or resolution.resolved_organization_name ilike '%' || term.term || '%'
where resolution.verification_status_code = 'verify_corroborated'
Comment on lines +89 to +97

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔍 verified_organization relies on exact entity_name string equality

The verified_organization CTE joins corporate_entity to the resolution cache with entity.entity_name = resolution.resolved_organization_name (global_ask_retrieval.py). This exact string equality (case- and whitespace-sensitive) means any divergence between the corroborated canonical name and the catalog's entity_name silently yields no verified organization, disabling the feature for that org. This is consistent with the design assumption that resolve_corporate_entity substitutes the canonical name so the catalog row already carries the exact canonical form, but it is worth confirming that assumption holds for all corroborated rows.

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

Comment on lines +89 to +97

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔍 verified_organization joins corporate_entity by non-unique display name

verified_organization joins corporate_entity on entity.entity_name = resolution.resolved_organization_name (global_ask_retrieval.py). Per the schema comment (migrations/0001_initial_schema.sql:446-447), corporate_entity.entity_name is not unique, so a corroborated resolution to a shared display name will fan out to every entity carrying that name and nominate all of their posts. Because nomination is explicitly non-authoritative and the caller re-runs the visibility predicate, this over-nomination is not an access/correctness bug, but it can broaden candidate sets beyond the intended single entity. Worth confirming this matches the ADR 0008 intent that the projection resolves back to a single canonical corporate entity.

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

Comment on lines +89 to +97

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Info: verified_organization CTE correctly gated to corroborated rows only

The verified_organization CTE inner-joins query_terms, so when the question yields no lexical terms (only ontology codes), the CTE is empty and the two new branches nominate nothing — matching the docstring. The verification_status_code = 'verify_corroborated' filter (verified against migrations/0004_relation_verification.sql:16) correctly excludes verify_pending/verify_uncorroborated rows. The ILIKE join predicates are backed by the new pg_trgm GIN indexes in migration 0055. No correctness issue found.

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

), candidate_post as (
select mention.post_id, post.created_at
from post_project_mention mention
Expand Down Expand Up @@ -126,6 +137,20 @@ async def semantic_candidate_post_ids(
where entity.entity_name ilike '%' || term.term || '%'
)
union all
select mention.post_id, post.created_at
from post_organization_mention mention
join verified_organization organization
on organization.corporate_entity_id = mention.corporate_entity_id
join source_post post on post.post_id = mention.post_id
union all
select mention.post_id, post.created_at
from post_person_mention mention
join person_affiliation affiliation
on affiliation.person_id = mention.person_id
join verified_organization organization
on organization.corporate_entity_id = affiliation.affiliated_corporate_entity_id
join source_post post on post.post_id = mention.post_id
Comment on lines +140 to +152

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Info: Verified-org union branches intentionally omit a query_terms predicate

The two new union branches (post_organization_mention via verified_organization, and post_person_mention via person_affiliation -> verified_organization) carry no exists (select 1 from query_terms ...) filter, unlike every other branch. This is by design: term matching happens once inside verified_organization (global_ask_retrieval.py), so the branches nominate all posts mentioning a verified entity / an affiliated person. Note the affiliation branch nominates every post mentioning an affiliated person regardless of whether that post concerns the organization; this matches the stated intent ("affiliated people's posts") but meaningfully widens candidate breadth compared to the other predicates.

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

union all
select mention.post_id, post.created_at
from post_team_mention mention
join cataloged_team team on team.team_id = mention.team_id
Expand Down
2 changes: 1 addition & 1 deletion docker/postgres-init/migrate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do
migration_name=${migration##*/}
case "$migration_name" in
0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;;
0051_*|0052_*|0053_*|0054_*) ;;
0051_*|0052_*|0053_*|0054_*|0055_*) ;;
0060_*|0100_*|0101_*|0102_*) ;;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔍 Migration 0103 absent from migrate.sh replay window

The case statement was extended to cover 0055_* (migrate.sh) but migrations/0103_tenant_settings.sql present in the repo is not matched by any case arm (line 22 stops at 0102_*). This is pre-existing and outside this PR's scope, but the same replay-window mechanism this PR relies on would silently skip 0103 on existing volumes.

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

*) continue ;;
esac
Expand Down
16 changes: 16 additions & 0 deletions docs/adr/0008-organization-abbreviation-resolution.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ different, complementary standard from ADR 0006/0007's PROV-O/ORG
classes: SKOS here labels the *string identity* relationship between
two names for the same thing, not the *type* of the named actor.

Global Ask treats only search-corroborated rows as a multilingual label
projection. A raw abbreviation, local-language name, or translated name that
has actually appeared in a post context can therefore nominate the same posts
as its canonical corporate-entity name. The projection joins the corroborated
`resolved_organization_name` back to `corporate_entity`; pending and
uncorroborated rows remain invisible. It does not generate translations or
infer aliases at query time. This preserves the source-observed label and the
SKOS preferred/alternative-label distinction while applying the document-level
context required by multilingual entity linking (De Cao et al., 2022).

Only a search-corroborated resolution is ever substituted in for
downstream entity matching (`resolve_corporate_entity`) -- an
LLM-proposed name with no corroboration, or with verification itself
Expand Down Expand Up @@ -111,6 +121,10 @@ canonical form too rather than reintroducing the raw abbreviation.
- Context-sensitive caching follows entity-linking evidence that ambiguous
mentions must be disambiguated with document-level semantic context, not a
name-only lookup (Rama-Maneiro, Vidal, & Lama, 2020).
- Search can cross language and abbreviation boundaries only after the existing
contextual-orchestrator plus SearXNG evidence path corroborates that label
pair. An unseen or unverified translation remains unavailable rather than
becoming a guessed catalog alias.

## Related

Expand All @@ -126,6 +140,8 @@ Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization s

Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304

De Cao, N., Wu, L., Popat, K., Artetxe, M., Goyal, N., Plekhanov, M., Zettlemoyer, L., & Riedel, S. (2022). Multilingual autoregressive entity linking. *Transactions of the Association for Computational Linguistics, 10*, 274–290. https://doi.org/10.1162/tacl_a_00460

Rama-Maneiro, E., Vidal, J. C., & Lama, M. (2020). Collective disambiguation in entity linking based on topic coherence in semantic graphs. *Knowledge-Based Systems, 199*, Article 105967. https://doi.org/10.1016/j.knosys.2020.105967

Thorne, J., Vlachos, A., Christodoulopoulos, C., & Mittal, A. (2018). FEVER: A large-scale dataset for fact extraction and VERification. In *Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies* (pp. 809–819). Association for Computational Linguistics. https://doi.org/10.18653/v1/N18-1074
22 changes: 21 additions & 1 deletion docs/doctoring/GLOBAL_ASK_SEMANTIC_SEARCH_REFERENCES.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@ column has its own `pg_trgm` GIN index, and the retrieval SQL keeps one direct
`concat_ws(...)` or another expression, because such a query would not match the
column indexes declared by migration 0054.

Organization names cross abbreviation and language boundaries only through
corroborated `organization_name_resolution` rows. The source-observed raw name
acts as a SKOS-style alternative label for the canonical corporate-entity name;
pending or uncorroborated mappings never nominate a post. This reuses the
existing contextual-orchestrator plus SearXNG evidence path and document context
rather than generating speculative translations at query time. Multilingual
entity-linking evidence supports using document context to connect surface forms
across languages (De Cao et al., 2022).

PostgreSQL documents that the `pg_trgm` GiST and GIN operator classes support
indexed `LIKE` and `ILIKE` searches even when a pattern is not left-anchored.
It also notes that patterns with no extractable trigrams can degenerate to a
Expand All @@ -20,11 +29,22 @@ Evidence in this repository:

- `backend/app/global_ask_retrieval.py`
- `migrations/0054_global_ask_semantic_search.sql`
- `migrations/0055_verified_organization_label_search.sql`
- `migrations/rollback/0054_global_ask_semantic_search.sql`
- `migrations/rollback/0055_verified_organization_label_search.sql`
- `tests/test_global_ask_retrieval.py`
- `tests/test_global_ask_semantic_indexes.py`

## APA 7th reference
## APA 7th references

De Cao, N., Wu, L., Popat, K., Artetxe, M., Goyal, N., Plekhanov, M.,
Zettlemoyer, L., & Riedel, S. (2022). Multilingual autoregressive entity
linking. *Transactions of the Association for Computational Linguistics, 10*,
274–290. https://doi.org/10.1162/tacl_a_00460

Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization
system reference*. World Wide Web Consortium.
https://www.w3.org/TR/skos-reference/

PostgreSQL Global Development Group. (2026). *pg_trgm—Support for similarity of
text using trigram matching* (PostgreSQL 17 documentation, Section F.33).
Expand Down
13 changes: 13 additions & 0 deletions migrations/0055_verified_organization_label_search.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
begin;

-- ADR 0008: only search-corroborated raw/canonical pairs act as Global Ask
-- aliases. These column indexes preserve multilingual contains-search without
-- copying context-scoped labels into a second table.
create extension if not exists pg_trgm;

create index if not exists organization_name_resolution_raw_search_idx
on organization_name_resolution using gin (raw_organization_name gin_trgm_ops);
create index if not exists organization_name_resolution_resolved_search_idx
on organization_name_resolution using gin (resolved_organization_name gin_trgm_ops);

commit;
Comment thread
seonghobae marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
begin;

drop index if exists organization_name_resolution_resolved_search_idx;
drop index if exists organization_name_resolution_raw_search_idx;

-- pg_trgm is shared with the broader Global Ask search slice.
commit;
5 changes: 5 additions & 0 deletions tests/test_global_ask_retrieval.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,11 @@ async def test_semantic_candidate_post_ids_is_bounded_deduplicated_and_indexable
assert "post_summary_role" in query
assert "post_person_mention" in query
assert "post_organization_mention" in query
assert "organization_name_resolution" in query
assert "resolution.verification_status_code = 'verify_corroborated'" in query
assert "resolution.raw_organization_name ilike" in query
assert "resolution.resolved_organization_name ilike" in query
assert "person_affiliation" in query
assert "post_team_mention" in query
assert "knowledge_graph_edge_evidence" in query

Expand Down
16 changes: 16 additions & 0 deletions tests/test_global_ask_semantic_indexes.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
ROOT = Path(__file__).resolve().parents[1]
FORWARD = ROOT / "migrations/0054_global_ask_semantic_search.sql"
ROLLBACK = ROOT / "migrations/rollback/0054_global_ask_semantic_search.sql"
ORGANIZATION_FORWARD = ROOT / "migrations/0055_verified_organization_label_search.sql"
ORGANIZATION_ROLLBACK = ROOT / "migrations/rollback/0055_verified_organization_label_search.sql"


EXPECTED_INDEXES = (
Expand Down Expand Up @@ -44,3 +46,17 @@ def test_migration_runner_includes_the_semantic_search_slice() -> None:
"""Long-lived Compose databases apply the same index contract as fresh installs."""
migrate = (ROOT / "docker/postgres-init/migrate.sh").read_text(encoding="utf-8")
assert "0054_*" in migrate


def test_verified_organization_label_indexes_have_a_symmetric_rollback() -> None:
"""The alias-search slice can be removed without dropping shared pg_trgm."""
forward = ORGANIZATION_FORWARD.read_text(encoding="utf-8").casefold()
rollback = ORGANIZATION_ROLLBACK.read_text(encoding="utf-8").casefold()

for index_name in (
"organization_name_resolution_raw_search_idx",
"organization_name_resolution_resolved_search_idx",
):
assert f"create index if not exists {index_name}" in forward
assert f"drop index if exists {index_name}" in rollback
assert "drop extension" not in rollback
14 changes: 14 additions & 0 deletions tests/test_migration_replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@


def test_shared_metric_migration_does_not_narrow_later_report_dimensions() -> None:
"""The shared metric migration preserves later team and project dimensions."""
sql = (
Path(__file__).resolve().parents[1]
/ "migrations"
Expand Down Expand Up @@ -48,6 +49,7 @@ def test_migrate_sh_replays_context_scoped_name_cache_migration() -> None:


def test_migrate_sh_replays_global_ask_context_migration() -> None:
"""Existing volumes must receive the Global Ask context migration."""
script = (
Path(__file__).resolve().parents[1]
/ "docker"
Expand All @@ -56,3 +58,15 @@ def test_migrate_sh_replays_global_ask_context_migration() -> None:
).read_text(encoding="utf-8")

assert "0052_*" in script


def test_migrate_sh_replays_verified_organization_label_search_migration() -> None:
"""Existing volumes must receive multilingual organization search indexes."""
script = (
Path(__file__).resolve().parents[1]
/ "docker"
/ "postgres-init"
/ "migrate.sh"
).read_text(encoding="utf-8")

assert "0055_*" in script
Loading