diff --git a/backend/app/main.py b/backend/app/main.py index 1cd132363..8bda198ff 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -660,47 +660,40 @@ async def _post_filter_options( process_unit_ids: frozenset[str], ) -> tuple[list[dict[str, str]], list[dict[str, str]]]: """Return every authorized filter value, not only values on the current page.""" - visibility_sql = f""" - select distinct post.visibility_code as code, - coalesce(lookup.lookup_label, post.visibility_code) as label, + options_sql = f""" + select distinct option.lookup_category, option.code, + coalesce(lookup.lookup_label, option.code) as label, coalesce(lookup.display_order, 2147483647) as display_order from source_post post + cross join lateral ( + values ('post_visibility', post.visibility_code), + ('voc_type', post.voc_type_code) + ) as option(lookup_category, code) left join common_lookup_value lookup - on lookup.lookup_category = 'post_visibility' - and lookup.lookup_code = post.visibility_code + on lookup.lookup_category = option.lookup_category + and lookup.lookup_code = option.code where (post.visibility_code = 'public' or (post.corporate_entity_id::text = any($1::text[]) and (cardinality($2::text[]) = 0 or post.process_unit_id::text = any($2::text[])))) and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} - order by display_order, code + order by option.lookup_category, display_order, option.code """ - type_sql = f""" - select distinct post.voc_type_code as code, - coalesce(lookup.lookup_label, post.voc_type_code) as label, - coalesce(lookup.display_order, 2147483647) as display_order - from source_post post - left join common_lookup_value lookup - on lookup.lookup_category = 'voc_type' - and lookup.lookup_code = post.voc_type_code - where (post.visibility_code = 'public' - or (post.corporate_entity_id::text = any($1::text[]) - and (cardinality($2::text[]) = 0 - or post.process_unit_id::text = any($2::text[])))) - and {SOURCE_POST_ELIGIBILITY_SQL.format(alias='post')} - order by display_order, code - """ - # Safe SQL: both query strings are closed lookup statements; entity ids remain asyncpg parameters. - visibility_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli - visibility_sql, list(corporate_entity_ids), list(process_unit_ids) - ) - # Safe SQL: both query strings are closed lookup statements; entity ids remain asyncpg parameters. - type_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli - type_sql, list(corporate_entity_ids), list(process_unit_ids) + # Safe SQL: this is a closed lookup statement; entity ids remain asyncpg parameters. + option_rows = await conn.fetch( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + options_sql, list(corporate_entity_ids), list(process_unit_ids) ) return ( - [{"code": row["code"], "label": row["label"]} for row in type_rows], - [{"code": row["code"], "label": row["label"]} for row in visibility_rows], + [ + {"code": row["code"], "label": row["label"]} + for row in option_rows + if row["lookup_category"] == "voc_type" + ], + [ + {"code": row["code"], "label": row["label"]} + for row in option_rows + if row["lookup_category"] == "post_visibility" + ], ) diff --git a/docs/adr/0212-single-query-authorized-post-filter-options.md b/docs/adr/0212-single-query-authorized-post-filter-options.md new file mode 100644 index 000000000..002cc5d58 --- /dev/null +++ b/docs/adr/0212-single-query-authorized-post-filter-options.md @@ -0,0 +1,41 @@ +# ADR 0212: Single-query authorized post-filter options + +- Status: Accepted +- Date: 2026-08-25 + +## Context + +`GET /api/posts` must return every visibility and VOC-type option represented +in the caller's authorized, current source-post population, not merely values +on the requested page. The projection applied the same ABAC and source-post +eligibility predicate in two sequential `SELECT DISTINCT` queries. An +application-ready local stack with 43,189 aggregate `source_post` rows showed +repeated filter-option queries active in PostgreSQL while `/api/posts` exceeded +30 seconds. That stack used an older backend image, so this is diagnostic +evidence for the query shape, not PR-head latency evidence or a product SLO. + +## Decision + +Project both lookup dimensions from one ABAC-filtered `source_post` relation +using a lateral two-row value projection, then deduplicate by lookup category +and code. Join `common_lookup_value` by both category and code, and partition +the result into the existing response fields in application code. + +The query keeps the complete authorized population and the existing public, +corporate-entity, process-unit, and source-eligibility predicates. It does not +derive options from the paginated result and does not cache options across +principals. No latency threshold is adopted; capacity remains an observed +property of a named environment and workload. + +## Consequences + +- Each post-list request performs one database query and round trip for both + option dimensions instead of two sequential queries and round trips. The + physical scan plan is not asserted until an exact-head `EXPLAIN` is recorded. +- Filter completeness and ABAC semantics remain unchanged. +- A focused unit test guards the one-query contract and bound ABAC parameters; + the existing authenticated integration test continues to guard visible + posts and complete option values. +- Runtime comparison still requires an exact-head application image and the + synthetic k6 procedure; older-image observations cannot establish the + improvement's latency effect. diff --git a/docs/adr/README.md b/docs/adr/README.md index a7704da7e..88ffb3abc 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,6 +18,7 @@ decision from them. | [`image-content-schema.md`](../image-content-schema.md) | [0066](0066-position-preserving-image-content.md) | | [`storybook-inventory.md`](../storybook-inventory.md) | [0118](0118-uiux-standard-guide-v3-design-overhaul.md), [0184](0184-ontology-provenance-explorer.md) | | [`POSTGRESQL_CONCURRENCY_REFERENCES.md`](../doctoring/POSTGRESQL_CONCURRENCY_REFERENCES.md) | [0204](0204-analysis-run-short-transaction-delivery.md) | +| [`operability/http-concurrency-evidence.md`](../operability/http-concurrency-evidence.md) | [0204](0204-analysis-run-short-transaction-delivery.md), [0212](0212-single-query-authorized-post-filter-options.md) | | Evidence operations Dashboard (`/`) | [0206](0206-evidence-operations-dashboard.md) | | [`temporal-topic-context-influence-research.md`](../temporal-topic-context-influence-research.md) | [0210](0210-temporal-topic-context-influence-dashboard.md) | | [`python-mathematical-compute-boundary-audit.md`](../doctoring/python-mathematical-compute-boundary-audit.md) | [0208](0208-externalize-local-mathematical-compute.md) | diff --git a/docs/operability/http-concurrency-evidence.md b/docs/operability/http-concurrency-evidence.md index 9e846f7ba..8ac0885fb 100644 --- a/docs/operability/http-concurrency-evidence.md +++ b/docs/operability/http-concurrency-evidence.md @@ -58,3 +58,22 @@ steps ranged up to 292.3 seconds. No containers were running afterward, so no HTTP latency distribution was produced and no application bottleneck is claimed. This is local build-environment evidence only. Re-run the command above on an application-ready stack to obtain the product measurement. + +## Older-image diagnostic observation + +On 2026-08-25, an application-ready local Compose stack configured with four +worker VUs completed zero full iterations in two observations. In the second +30-second observation, Ask enqueue took 2.69 seconds, the maximum completed +HTTP request took 45.26 seconds, and k6 recorded no failures among requests +that completed. Isolated observations were: `/api/posts` did not complete +within 30 seconds, `/api/lineage` took 12.152 seconds, and Ask polling took +0.456 seconds. The database contained 43,189 aggregate `source_post` rows; +`pg_stat_activity` showed repeated post-filter `DISTINCT` queries active with +`MessageQueueSend` waits. + +The backend image was from an older branch, not the current or ADR 0212 change +head. These aggregate, non-identifying values support investigating the +duplicate filter-option query; they do not demonstrate current-head latency, +causality, capacity, or an SLO. ADR 0212 combines the two option projections +into one database query; its physical plan remains to be measured exact-head. +Repeat the synthetic k6 run on an exact-head image before comparing effects. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d922aca0e..9a65c2eb6 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,7 +1,7 @@ # Product & Technical Gap Baseline -> Dashboard delivery snapshot: 2026-08-25 20:41 KST. Protected `main` was -> `48f013a28e0b0fe51951d4df9bf3f9a3532df173`. This local branch is not +> Dashboard delivery snapshot: 2026-08-25 21:34 KST. Protected `main` was +> `d7d5eeb310b055b5e138060cf2dfb929b03090a6`. This local branch is not > protected-main release evidence. ## Operations Dashboard PRD/TRD traceability @@ -44,7 +44,7 @@ tables use composite keys and bounded kind-first indexes; production hot-path acceptance still requires `EXPLAIN (ANALYZE, BUFFERS)` on an anonymized runtime snapshot. -### Exact-head UI audit +### Historical UI audit evidence The `f0b96029` Storybook build was rendered at 1440×1100 and 402×1200 with synthetic evidence; `416fd19d` changes only post-navigation request isolation. @@ -60,18 +60,18 @@ only aggregate, non-identifying evidence to this repository. ### Exact open-PR boundary -At this snapshot there were 5 open PRs and 14 open issues. Exact observed heads -were `#621 cca022a8` (this PR's observed parent), `#620 4bc7aeac`, -`#619 29e01dc2`, `#618 df9d6cb1`, and `#387 ceb2c1d4`. PR #579 closed without -merge and is not protected delivery. -PRs #612, #614, #615, and #616 reached protected `main`; the superseded baseline -PR #613 closed without merge and its PRD is recreated by this branch. The open -heads remain blocked on hosted gates and/or independent review. These +At this snapshot there were 3 open PRs and 11 open issues. Exact observed heads +were `#628 d07d212f` (this branch's observed parent), `#627 9e0528a6`, and +`#579 1c209c85`. PR #579 is open; its ADR 0211 reservation is why this branch's +filter-option decision is ADR 0212. PRs #612, #614, #615, #616, and #626 +reached protected `main`; the superseded baseline PR #613 closed without merge +and its PRD was recreated on protected main. The open heads remain blocked on +hosted gates and/or independent review. These observations are not merge readiness. Re-fetch exact heads, unresolved threads, checks, approvals, rulesets, and merge SHA before any lifecycle claim. -> Audit snapshot: 2026-08-25 20:41 KST (refreshed by the autonomous merge +> Audit snapshot: 2026-08-25 21:34 KST (refreshed by the autonomous merge > loop). This repository records synthetic fixtures and aggregate, > non-identifying runtime evidence only. Open PRs and local checks are not > protected-default-branch release evidence. Identifying post identifiers, @@ -80,19 +80,17 @@ lifecycle claim. ## 1. Exact-head and governance evidence -The protected default branch was `48f013a28e0b0fe51951d4df9bf3f9a3532df173` -when this baseline was refreshed. The live queue contained 5 open PRs and 14 +The protected default branch was `d7d5eeb310b055b5e138060cf2dfb929b03090a6` +when this baseline was refreshed. The live queue contained 3 open PRs and 11 open issues. The exact-head inventory below supersedes older per-PR snapshots elsewhere in this document; those older rows remain useful historical delivery context only. | PR | Exact observed head | Merge/check state at this snapshot | | ---: | --- | --- | -| #621 | `cca022a8` (observed parent) | this row is written by #621 itself, so its commit necessarily advances after the snapshot is encoded; re-fetch the live head before governance use | -| #620 | `4bc7aeac` | refreshes temporal-topic and capacity gaps; hosted gates and independent review remain required | -| #619 | `29e01dc2` | documents the fixed-alias, repository-owned Similar-VOC eligibility fragment for the narrow Semgrep rule; focused tests and Semgrep passed, while hosted gates and independent review remain required | -| #618 | `df9d6cb1` | separates SHACL class/property term kinds and corrects the corporate-entity UI fixture; current `main` is composed, while hosted gates and independent review remain required | -| #387 | `ceb2c1d4` | channel evidence is composed with protected main and focused tests pass, but independent exact-head review remains required | +| #628 | `d07d212f` (observed parent) | this row is updated by #628 itself, so its exact head advances after the snapshot is encoded; ADR 0212 combines complete ABAC-visible filter options into one database round trip, while hosted gates and independent review remain required | +| #627 | `9e0528a6` | repairs k6 lifecycle evidence preservation; hosted gates remain required | +| #579 | `1c209c85` | persists leftover interaction-map coordinates and owns ADR 0211; hosted gates and independent review remain required | No row above is merge evidence. Immediately before any lifecycle action, re-fetch the head, unresolved threads, formal reviews, rulesets, and same-head @@ -364,12 +362,12 @@ this file per §3.5 of the prior snapshot). | Gap | Current evidence | Acceptance requirement | | --- | --- | --- | -| Protected release | 5 open PRs at snapshot; #618–#621 are current-main follow-ups, while #387 retains independent-review gates and #579 closed without merge | Terminal exact-head checks, no unresolved threads, independent exact-head approvals, protected squash-merge SHA | +| Protected release | 3 open PRs at snapshot: #627 and #628 are current-main performance follow-ups, while reopened #579 retains hosted and independent-review gates | Terminal exact-head checks, no unresolved threads, independent exact-head approvals, protected squash-merge SHA | | Evidence-grounded operations workspace | Protected-main #614 delivers governed semantic Ask, live Similar VOC, disjoint pending/failed analysis metrics, full Storybook state inventory, and current desktop/mobile screenshot evidence. Authorized-corpus backfill acceptance remains unavailable | Perform authenticated authorized-corpus acceptance with aggregate evidence and retain fail-closed no-match behavior | | Shared frontend gate | The ADR 0109 login repair is on protected `main`; eight older branches carried the defect and received the same verified repair this loop (#521–#560) | Keep every future branch cut from post-repair bases; re-verify with frontend lint/test/build before push | | Identifying baseline regression | `main` gap file listed real post identifiers; separately, closed #506 and pre-existing public history contain a private runtime source-table identifier, while current `main` and #507 trees are clean | Land this non-identifying rewrite, then coordinate ADR 0001 history remediation with security/privacy owners; do not reproduce the value, force-push, or delete evidence ad hoc | | Authorized-corpus runtime | Repository tests use synthetic fixtures; private records remain outside git | Authenticated runtime validation returning only aggregate, non-identifying evidence | -| Concurrent web responsiveness | ADR 0204 releases pooled transactions during provider work, and the synthetic Compose boundary now has an authenticated k6 E2E harness for Ask enqueue, concurrent reads, and job polling. No environment-specific measurement is a protected-main product guarantee | Run `make load-http` with declared environment concurrency/window; retain raw distributions and resource configuration, then diagnose bottlenecks with backend, PostgreSQL, Valkey, and orchestrator evidence before setting any approved SLO | +| Concurrent web responsiveness | ADR 0204 releases pooled transactions during provider work, and the synthetic Compose boundary has an authenticated k6 E2E harness for Ask enqueue, concurrent reads, and job polling. An older-image local observation found repeated post-filter queries while `/api/posts` exceeded 30 seconds; ADR 0212 combines two authorized filter-option queries into one round trip without narrowing ABAC-visible options. The observation is not exact-head evidence or a product guarantee, and no physical scan reduction is claimed without an exact-head plan | Rebuild an exact-head application image, run `make load-http` with declared environment concurrency/window, and retain raw distributions and resource configuration. Compare the post-list database plan and latency with ADR 0212 while preserving the complete authorized filter set; set no SLO until representative capacity evidence is approved | | Image understanding | Region, OCR, and description work exists across active heads (#405, #419), but current runtime acceptance has not yet proved table-image structure, complete region coverage, or summary/image readiness together | Orchestrator-backed rendered workflow, original/derived asset provenance, region-before-OCR processing, and honest unsupported states; reconcile ADR 0052's image-bearing summary readiness with ADR 0098 before changing sequencing | | Semantic source rendering | Paragraph, table, list, formula, and indentation work exists across stacks (#394, #427, #448–#450); #515 adds synthetic backend/frontend parity for deterministic rows/cells, footnote boundaries, and encoded scripts | Land the #427 → #515 stack, then gather authenticated browser evidence that list nesting, continuation alignment, and formula units render without authoring-layout artifacts | | Event and project semantics | Multi-project mentions, project-bound actions, 5W1H, requester/processor, and semantic relations exist in ADR 0036/0052/0100/0111/0129 and active stacks | Aggregate authenticated evidence must show distinct projects and events, explicit requester/processor and real R&R, normalized relative time, and product/entity relations without promoting attendance or co-occurrence | diff --git a/tests/test_post_filter_options.py b/tests/test_post_filter_options.py new file mode 100644 index 000000000..076f0ec25 --- /dev/null +++ b/tests/test_post_filter_options.py @@ -0,0 +1,64 @@ +"""Unit tests for the authorized post-filter projection.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from backend.app.main import _post_filter_options + + +class _RecordingConnection: + """Return synthetic option rows while recording database round trips.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + + async def fetch(self, query: str, *args: Any) -> list[dict[str, object]]: + """Record the closed query and return both supported option categories.""" + self.calls.append((query, args)) + return [ + { + "lookup_category": "post_visibility", + "code": "public", + "label": "Public", + "display_order": 1, + }, + { + "lookup_category": "post_visibility", + "code": "private", + "label": "Private", + "display_order": 2, + }, + { + "lookup_category": "voc_type", + "code": "voc", + "label": "Voice of Customer", + "display_order": 1, + }, + ] + + +def test_post_filter_options_use_one_authorized_source_scan() -> None: + """Both complete option lists share one parameterized ABAC-filtered query.""" + conn = _RecordingConnection() + + voc_types, visibilities = asyncio.run( + _post_filter_options(conn, frozenset({"corp-a"}), frozenset({"pu-a"})) + ) + + assert voc_types == [{"code": "voc", "label": "Voice of Customer"}] + assert visibilities == [ + {"code": "public", "label": "Public"}, + {"code": "private", "label": "Private"}, + ] + assert len(conn.calls) == 1 + query, args = conn.calls[0] + assert "cross join lateral" in query + assert "('post_visibility', post.visibility_code)" in query + assert "('voc_type', post.voc_type_code)" in query + assert "post.corporate_entity_id::text = any($1::text[])" in query + assert "post.process_unit_id::text = any($2::text[])" in query + assert "nullif(btrim(post.source_draft_code), '') is null" in query + assert "nullif(btrim(post.source_deleted_flag), '') is null" in query + assert args == (["corp-a"], ["pu-a"])