diff --git a/backend/app/global_ask_retrieval.py b/backend/app/global_ask_retrieval.py index 9d35b6a4a..b0b333789 100644 --- a/backend/app/global_ask_retrieval.py +++ b/backend/app/global_ask_retrieval.py @@ -68,10 +68,12 @@ async def semantic_candidate_post_ids( Project mentions, responsibility/affiliation evidence, Keyman names, and organization/team catalogs use one ``ILIKE`` predicate per indexed column. - This preserves multilingual substring lookup without wrapping indexed fields - in an expression that forces a sequential scan. Ontology lookup codes are - applied only to graph lookup-code columns. The function never returns source - text and nomination never grants access. + Search-corroborated raw/canonical organization-name pairs additionally + nominate direct organization mentions and affiliated people's posts; pending + or uncorroborated aliases do not. This preserves multilingual substring lookup + without wrapping indexed fields in an expression that forces a sequential + scan. Ontology lookup codes are applied only to graph lookup-code columns. + The function never returns source text and nomination never grants access. """ if maximum_candidates <= 0: @@ -84,6 +86,15 @@ async def semantic_candidate_post_ids( """ with query_terms as ( select unnest($1::text[]) as term + ), verified_organization as ( + select distinct entity.corporate_entity_id + from organization_name_resolution resolution + join corporate_entity entity + on entity.entity_name = resolution.resolved_organization_name + join query_terms term + on resolution.raw_organization_name ilike '%' || term.term || '%' + or resolution.resolved_organization_name ilike '%' || term.term || '%' + where resolution.verification_status_code = 'verify_corroborated' ), candidate_post as ( select mention.post_id, post.created_at from post_project_mention mention @@ -126,6 +137,20 @@ async def semantic_candidate_post_ids( where entity.entity_name ilike '%' || term.term || '%' ) union all + select mention.post_id, post.created_at + from post_organization_mention mention + join verified_organization organization + on organization.corporate_entity_id = mention.corporate_entity_id + join source_post post on post.post_id = mention.post_id + union all + select mention.post_id, post.created_at + from post_person_mention mention + join person_affiliation affiliation + on affiliation.person_id = mention.person_id + join verified_organization organization + on organization.corporate_entity_id = affiliation.affiliated_corporate_entity_id + join source_post post on post.post_id = mention.post_id + union all select mention.post_id, post.created_at from post_team_mention mention join cataloged_team team on team.team_id = mention.team_id diff --git a/docker/postgres-init/migrate.sh b/docker/postgres-init/migrate.sh index 712107b27..422abe04c 100644 --- a/docker/postgres-init/migrate.sh +++ b/docker/postgres-init/migrate.sh @@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do migration_name=${migration##*/} case "$migration_name" in 0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;; - 0051_*|0052_*|0053_*|0054_*) ;; + 0051_*|0052_*|0053_*|0054_*|0055_*) ;; 0060_*|0100_*|0101_*|0102_*) ;; *) continue ;; esac diff --git a/docs/adr/0008-organization-abbreviation-resolution.md b/docs/adr/0008-organization-abbreviation-resolution.md index 59e4e8717..21a08dc0d 100644 --- a/docs/adr/0008-organization-abbreviation-resolution.md +++ b/docs/adr/0008-organization-abbreviation-resolution.md @@ -56,6 +56,16 @@ different, complementary standard from ADR 0006/0007's PROV-O/ORG classes: SKOS here labels the *string identity* relationship between two names for the same thing, not the *type* of the named actor. +Global Ask treats only search-corroborated rows as a multilingual label +projection. A raw abbreviation, local-language name, or translated name that +has actually appeared in a post context can therefore nominate the same posts +as its canonical corporate-entity name. The projection joins the corroborated +`resolved_organization_name` back to `corporate_entity`; pending and +uncorroborated rows remain invisible. It does not generate translations or +infer aliases at query time. This preserves the source-observed label and the +SKOS preferred/alternative-label distinction while applying the document-level +context required by multilingual entity linking (De Cao et al., 2022). + Only a search-corroborated resolution is ever substituted in for downstream entity matching (`resolve_corporate_entity`) -- an LLM-proposed name with no corroboration, or with verification itself @@ -111,6 +121,10 @@ canonical form too rather than reintroducing the raw abbreviation. - Context-sensitive caching follows entity-linking evidence that ambiguous mentions must be disambiguated with document-level semantic context, not a name-only lookup (Rama-Maneiro, Vidal, & Lama, 2020). +- Search can cross language and abbreviation boundaries only after the existing + contextual-orchestrator plus SearXNG evidence path corroborates that label + pair. An unseen or unverified translation remains unavailable rather than + becoming a guessed catalog alias. ## Related @@ -126,6 +140,8 @@ Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization s Bhattacharya, I., & Getoor, L. (2007). Collective entity resolution in relational data. *ACM Transactions on Knowledge Discovery from Data*, 1(1), Article 5. https://doi.org/10.1145/1217299.1217304 +De Cao, N., Wu, L., Popat, K., Artetxe, M., Goyal, N., Plekhanov, M., Zettlemoyer, L., & Riedel, S. (2022). Multilingual autoregressive entity linking. *Transactions of the Association for Computational Linguistics, 10*, 274–290. https://doi.org/10.1162/tacl_a_00460 + Rama-Maneiro, E., Vidal, J. C., & Lama, M. (2020). Collective disambiguation in entity linking based on topic coherence in semantic graphs. *Knowledge-Based Systems, 199*, Article 105967. https://doi.org/10.1016/j.knosys.2020.105967 Thorne, J., Vlachos, A., Christodoulopoulos, C., & Mittal, A. (2018). FEVER: A large-scale dataset for fact extraction and VERification. In *Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies* (pp. 809–819). Association for Computational Linguistics. https://doi.org/10.18653/v1/N18-1074 diff --git a/docs/doctoring/GLOBAL_ASK_SEMANTIC_SEARCH_REFERENCES.md b/docs/doctoring/GLOBAL_ASK_SEMANTIC_SEARCH_REFERENCES.md index c05aaf203..666d94c89 100644 --- a/docs/doctoring/GLOBAL_ASK_SEMANTIC_SEARCH_REFERENCES.md +++ b/docs/doctoring/GLOBAL_ASK_SEMANTIC_SEARCH_REFERENCES.md @@ -9,6 +9,15 @@ column has its own `pg_trgm` GIN index, and the retrieval SQL keeps one direct `concat_ws(...)` or another expression, because such a query would not match the column indexes declared by migration 0054. +Organization names cross abbreviation and language boundaries only through +corroborated `organization_name_resolution` rows. The source-observed raw name +acts as a SKOS-style alternative label for the canonical corporate-entity name; +pending or uncorroborated mappings never nominate a post. This reuses the +existing contextual-orchestrator plus SearXNG evidence path and document context +rather than generating speculative translations at query time. Multilingual +entity-linking evidence supports using document context to connect surface forms +across languages (De Cao et al., 2022). + PostgreSQL documents that the `pg_trgm` GiST and GIN operator classes support indexed `LIKE` and `ILIKE` searches even when a pattern is not left-anchored. It also notes that patterns with no extractable trigrams can degenerate to a @@ -20,11 +29,22 @@ Evidence in this repository: - `backend/app/global_ask_retrieval.py` - `migrations/0054_global_ask_semantic_search.sql` +- `migrations/0055_verified_organization_label_search.sql` - `migrations/rollback/0054_global_ask_semantic_search.sql` +- `migrations/rollback/0055_verified_organization_label_search.sql` - `tests/test_global_ask_retrieval.py` - `tests/test_global_ask_semantic_indexes.py` -## APA 7th reference +## APA 7th references + +De Cao, N., Wu, L., Popat, K., Artetxe, M., Goyal, N., Plekhanov, M., +Zettlemoyer, L., & Riedel, S. (2022). Multilingual autoregressive entity +linking. *Transactions of the Association for Computational Linguistics, 10*, +274–290. https://doi.org/10.1162/tacl_a_00460 + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge organization +system reference*. World Wide Web Consortium. +https://www.w3.org/TR/skos-reference/ PostgreSQL Global Development Group. (2026). *pg_trgm—Support for similarity of text using trigram matching* (PostgreSQL 17 documentation, Section F.33). diff --git a/migrations/0055_verified_organization_label_search.sql b/migrations/0055_verified_organization_label_search.sql new file mode 100644 index 000000000..d8b2f09d7 --- /dev/null +++ b/migrations/0055_verified_organization_label_search.sql @@ -0,0 +1,13 @@ +begin; + +-- ADR 0008: only search-corroborated raw/canonical pairs act as Global Ask +-- aliases. These column indexes preserve multilingual contains-search without +-- copying context-scoped labels into a second table. +create extension if not exists pg_trgm; + +create index if not exists organization_name_resolution_raw_search_idx + on organization_name_resolution using gin (raw_organization_name gin_trgm_ops); +create index if not exists organization_name_resolution_resolved_search_idx + on organization_name_resolution using gin (resolved_organization_name gin_trgm_ops); + +commit; diff --git a/migrations/rollback/0055_verified_organization_label_search.sql b/migrations/rollback/0055_verified_organization_label_search.sql new file mode 100644 index 000000000..1cef1add6 --- /dev/null +++ b/migrations/rollback/0055_verified_organization_label_search.sql @@ -0,0 +1,7 @@ +begin; + +drop index if exists organization_name_resolution_resolved_search_idx; +drop index if exists organization_name_resolution_raw_search_idx; + +-- pg_trgm is shared with the broader Global Ask search slice. +commit; diff --git a/tests/test_global_ask_retrieval.py b/tests/test_global_ask_retrieval.py index f514425d8..21a3bc33f 100644 --- a/tests/test_global_ask_retrieval.py +++ b/tests/test_global_ask_retrieval.py @@ -127,6 +127,11 @@ async def test_semantic_candidate_post_ids_is_bounded_deduplicated_and_indexable assert "post_summary_role" in query assert "post_person_mention" in query assert "post_organization_mention" in query + assert "organization_name_resolution" in query + assert "resolution.verification_status_code = 'verify_corroborated'" in query + assert "resolution.raw_organization_name ilike" in query + assert "resolution.resolved_organization_name ilike" in query + assert "person_affiliation" in query assert "post_team_mention" in query assert "knowledge_graph_edge_evidence" in query diff --git a/tests/test_global_ask_semantic_indexes.py b/tests/test_global_ask_semantic_indexes.py index 7e28f9573..1278d7e00 100644 --- a/tests/test_global_ask_semantic_indexes.py +++ b/tests/test_global_ask_semantic_indexes.py @@ -6,6 +6,8 @@ ROOT = Path(__file__).resolve().parents[1] FORWARD = ROOT / "migrations/0054_global_ask_semantic_search.sql" ROLLBACK = ROOT / "migrations/rollback/0054_global_ask_semantic_search.sql" +ORGANIZATION_FORWARD = ROOT / "migrations/0055_verified_organization_label_search.sql" +ORGANIZATION_ROLLBACK = ROOT / "migrations/rollback/0055_verified_organization_label_search.sql" EXPECTED_INDEXES = ( @@ -44,3 +46,17 @@ def test_migration_runner_includes_the_semantic_search_slice() -> None: """Long-lived Compose databases apply the same index contract as fresh installs.""" migrate = (ROOT / "docker/postgres-init/migrate.sh").read_text(encoding="utf-8") assert "0054_*" in migrate + + +def test_verified_organization_label_indexes_have_a_symmetric_rollback() -> None: + """The alias-search slice can be removed without dropping shared pg_trgm.""" + forward = ORGANIZATION_FORWARD.read_text(encoding="utf-8").casefold() + rollback = ORGANIZATION_ROLLBACK.read_text(encoding="utf-8").casefold() + + for index_name in ( + "organization_name_resolution_raw_search_idx", + "organization_name_resolution_resolved_search_idx", + ): + assert f"create index if not exists {index_name}" in forward + assert f"drop index if exists {index_name}" in rollback + assert "drop extension" not in rollback diff --git a/tests/test_migration_replay.py b/tests/test_migration_replay.py index 29098f0a5..e2b928422 100644 --- a/tests/test_migration_replay.py +++ b/tests/test_migration_replay.py @@ -2,6 +2,7 @@ def test_shared_metric_migration_does_not_narrow_later_report_dimensions() -> None: + """The shared metric migration preserves later team and project dimensions.""" sql = ( Path(__file__).resolve().parents[1] / "migrations" @@ -48,6 +49,7 @@ def test_migrate_sh_replays_context_scoped_name_cache_migration() -> None: def test_migrate_sh_replays_global_ask_context_migration() -> None: + """Existing volumes must receive the Global Ask context migration.""" script = ( Path(__file__).resolve().parents[1] / "docker" @@ -56,3 +58,15 @@ def test_migrate_sh_replays_global_ask_context_migration() -> None: ).read_text(encoding="utf-8") assert "0052_*" in script + + +def test_migrate_sh_replays_verified_organization_label_search_migration() -> None: + """Existing volumes must receive multilingual organization search indexes.""" + script = ( + Path(__file__).resolve().parents[1] + / "docker" + / "postgres-init" + / "migrate.sh" + ).read_text(encoding="utf-8") + + assert "0055_*" in script