From f488ff547bf3dd1dceb6a5b8a632823df9e2fa2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sun, 26 Apr 2026 18:24:33 +0100 Subject: [PATCH 1/5] feat(db)!: convert entities.entity_type to entity_type_id FK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the text-slug column on entities with an integer FK on entity_types(id). Two motivations folded into one change: 1. Integrity. The previous text-slug pattern silently orphaned entities when their entity_type was renamed or hard-deleted; only the create- path validator enforced same-org existence, and updates/deletes bypassed it. The FK now does that work at the DB layer: renames are safe (slug becomes display-only via JOIN), Postgres refuses to drop a referenced type. 2. Cross-org vocabulary. entity_types.id is globally unique, so an entity in tenant org A can carry a type defined in public-catalog org B by FK alone — without an extra org_id column on entities. Sets up the world-model plan in docs/plans/world-model.md for slim install and public-catalog references in subsequent PRs. Migration: db/migrations/20260426120000_entities_entity_type_fk.sql. Single-prod-DB: adds nullable column, backfills from existing slug lookup, fails loudly on orphaned rows, sets NOT NULL, drops the text column. RUN MANUALLY before deploying. All read sites switched to "JOIN entity_types et ON et.id = e.entity_type_id" and select "et.slug AS entity_type". INSERT sites resolve slug → id first. Test fixtures updated to the same pattern. The cross-org write guard for entity_relationships is deferred to the PR that introduces cross-org INSERTs (slim install + agent_template entity); adding it now would be dead code. --- ...20260426120000_entities_entity_type_fk.sql | 74 ++++ docs/plans/world-model.md | 331 ++++++++++++++++++ .../connectors/whatsapp-entity-links.test.ts | 20 +- .../entities/member-email-redaction.test.ts | 4 +- .../src/__tests__/setup/test-fixtures.ts | 19 +- .../src/auth/subject-identities.ts | 12 +- packages/owletto-backend/src/index.ts | 5 +- packages/owletto-backend/src/public-pages.ts | 12 +- .../src/tools/admin/manage_classifiers.ts | 3 +- .../src/tools/admin/manage_entity.ts | 6 +- .../src/tools/admin/manage_entity_schema.ts | 20 +- .../src/tools/admin/manage_watchers.ts | 24 +- .../owletto-backend/src/tools/get_content.ts | 13 +- .../owletto-backend/src/tools/get_watchers.ts | 17 +- .../owletto-backend/src/tools/resolve_path.ts | 30 +- .../owletto-backend/src/tools/save_content.ts | 6 +- packages/owletto-backend/src/tools/search.ts | 13 +- .../__tests__/entity-link-upsert.test.ts | 47 ++- .../owletto-backend/src/utils/auto-linker.ts | 12 +- .../src/utils/entity-link-upsert.ts | 24 +- .../src/utils/entity-management.ts | 64 ++-- .../src/utils/event-kind-validation.ts | 2 +- .../src/utils/execute-data-sources.ts | 27 +- .../src/utils/member-entity.ts | 34 +- .../src/utils/relationship-validation.ts | 7 +- .../owletto-backend/src/utils/table-schema.ts | 2 + 26 files changed, 689 insertions(+), 139 deletions(-) create mode 100644 db/migrations/20260426120000_entities_entity_type_fk.sql create mode 100644 docs/plans/world-model.md diff --git a/db/migrations/20260426120000_entities_entity_type_fk.sql b/db/migrations/20260426120000_entities_entity_type_fk.sql new file mode 100644 index 000000000..dc3a5216f --- /dev/null +++ b/db/migrations/20260426120000_entities_entity_type_fk.sql @@ -0,0 +1,74 @@ +-- migrate:up + +-- Convert entities.entity_type from a text slug to an FK on entity_types(id). +-- Two motivations folded into one change: +-- +-- 1. Integrity. Today entity_types renames orphan all referencing entities +-- (slug-based reference is silent FK with no enforcement). Hard-deletes +-- bypass the validator entirely. With a real FK, Postgres refuses to +-- drop a referenced type and renames update for free (the slug becomes +-- display only — JOIN to entity_types for it). +-- +-- 2. Cross-org vocabulary. entity_types.id is globally unique (one sequence +-- across all orgs), so an entity in tenant org A can carry a type defined +-- in public-catalog org B by FK alone. No additional org_id column on +-- entities is needed once the slug-based same-org coupling is gone. +-- +-- Single-prod-DB migration: add nullable column, backfill, fail loudly on +-- orphans, set NOT NULL, drop the text column. Run manually. + +-- 1. Add the FK column, nullable for backfill. +ALTER TABLE public.entities + ADD COLUMN entity_type_id integer REFERENCES public.entity_types(id); + +-- 2. Backfill from existing (organization_id, entity_type slug) → entity_types.id. +-- Soft-deleted entity_types still resolve — preserves history of soft-removed types. +UPDATE public.entities e +SET entity_type_id = et.id +FROM public.entity_types et +WHERE et.slug = e.entity_type + AND et.organization_id = e.organization_id + AND e.entity_type_id IS NULL; + +-- 3. Fail loudly on orphans. If any entities reference a slug with no matching +-- entity_types row, that's pre-existing data corruption from the slug-based +-- regime. Surface it; don't paper over. +DO $$ +DECLARE + orphan_count integer; +BEGIN + SELECT COUNT(*) INTO orphan_count FROM public.entities WHERE entity_type_id IS NULL; + IF orphan_count > 0 THEN + RAISE EXCEPTION + 'entity_type FK migration: % entities have entity_type slugs with no matching entity_types row. Investigate before re-running.', + orphan_count; + END IF; +END $$; + +-- 4. Tighten the FK column. +ALTER TABLE public.entities + ALTER COLUMN entity_type_id SET NOT NULL; + +-- 5. Index for filter/list queries that previously used entity_type slug. +CREATE INDEX idx_entities_entity_type_id + ON public.entities (entity_type_id) + WHERE deleted_at IS NULL; + +-- 6. Drop the text column. All readers JOIN to entity_types for the slug. +ALTER TABLE public.entities DROP COLUMN entity_type; + + +-- migrate:down + +ALTER TABLE public.entities ADD COLUMN entity_type text; + +UPDATE public.entities e +SET entity_type = et.slug +FROM public.entity_types et +WHERE et.id = e.entity_type_id; + +ALTER TABLE public.entities ALTER COLUMN entity_type SET NOT NULL; + +DROP INDEX IF EXISTS public.idx_entities_entity_type_id; + +ALTER TABLE public.entities DROP COLUMN entity_type_id; diff --git a/docs/plans/world-model.md b/docs/plans/world-model.md new file mode 100644 index 000000000..58661c589 --- /dev/null +++ b/docs/plans/world-model.md @@ -0,0 +1,331 @@ +# World Model + +Long-term shape of how knowledge, identity, and templates are organized across +tenants and public catalogs in Lobu. + +## TL;DR + +- Two org kinds (`tenant`, `public_catalog`), two visibilities (`private`, `public`). +- One graph: `entities` + `entity_relationships` + `entity_identities` are the + universal primitives. Orgs are trust slices through that graph. +- Cross-org relationships allowed in **one direction only**: tenant → public_catalog. +- Templates are entities of type `agent_template` in a public_catalog org. No + schema cloning on install — agents read vocabularies from public catalogs at + runtime. +- Contribution to public knowledge happens by inviting the public org's admin + agent into your private org as a member. Existing membership/messaging + primitives, no draft tables, no contributor roles. +- No Postgres RLS in phase 1. App-level org-scoped queries (already in place) + plus a write-side guard on relationship inserts are sufficient given the + one-directional reference rule. + +## Cleanup before phase 1 + +| PR | Action | Why | +| --- | --- | --- | +| #351 — `managed_by_template_agent_id` + `source_template_org_id` columns | **Close** | No mirroring → no tracking columns | +| #353 — `installAgentFromTemplate` schema mirror (1221 LOC) | **Close** | Schema lives in public catalogs; agents reference, don't clone | +| #357 — POST /api/install | Trim to ~30 lines | Just inserts an agent row in tenant + provisions identity | +| #359 — identity provisioning ($member + wa_jid) | Keep | Orthogonal — identities are real regardless | +| #362 — install manifest | Trim | Drop the env-var slug→bot-phone map; bot phone moves to data on the template entity | + +Also revisit #358 (company-aware world model for personal-finance) against this +plan once it lands — its direction is compatible but its details may need to be +re-aligned. + +## Long-term primitives + +| Primitive | Purpose | Stable? | +| --- | --- | --- | +| `organization` (with `kind` + `visibility`) | Trust boundary | Yes | +| `entities` (typed rows, scoped to one org) | Anything: $member, a company, a tax filing, an agent template, a review | Yes | +| `entity_types` + `entity_relationship_types` | Vocabulary, **data not schema** — templates ship new types as rows, not migrations | Yes | +| `entity_relationships` (typed edges, explicit `source_organization_id` + `target_organization_id`) | All semantic facts, references, forks, reviews | Yes | +| `entity_identities` (namespace + identifier → entity) | Technical lookup keys (auth_user_id, email, wa_jid, uk_utr, uk_ni, companies-house-number) | Yes | + +UUIDs everywhere — federation across instances or third-party catalogs becomes +cheap to add later. + +## Org topology + +- **`tenant`** — user's private space. `visibility=private`. Personal data, + installed agents, filings, message history. +- **`public_catalog`** — curated public knowledge & published artifacts. + `visibility=public`. Companies, gov bodies, currencies, tax years, allowance + definitions, agent templates, skill definitions, reviews. + +Three kinds of orgs collapsed to two: there is no `template` org kind. Templates +are entities of type `agent_template` in some public_catalog org, distinguished +by entity type, not org kind. + +## Cross-org references + +- Direction: tenant → public_catalog only (one-way). +- Read paths never mix scopes: queries either hit the user's org + (membership-scoped) or public orgs (`visibility=public` filter), never both at + once. This is what removes the "every read site must remember `OR + visibility=public`" risk. +- Write-side guard at the application layer: when inserting an + `entity_relationship`, validate that `target_organization_id` is either the + same org as the source OR an org with `visibility='public'`. A Postgres + trigger version of the same check is cheap defense-in-depth if/when needed. +- RLS is **not required** for this model to be safe. It remains a sensible + defense-in-depth project for later but is decoupled from world-model + delivery. + +## Templates + +A template is an entity of type `agent_template`: + +- Carries: system prompt, model config, tool list, skill manifest, version, + bot phone, descriptive metadata. +- References public catalogs it operates over via relationships: + `uses_catalog` → `public-uk-tax`, `uses_catalog` → `public-uk-finance`. +- Authorship, forks, reviews, ratings: `entity_relationships` (`authored_by`, + `forked_from`, `reviews`, `rated`). + +Installation: + +1. Insert agent row in user's tenant org with `template_entity_id`. +2. Provision `$member` if missing (identity provisioning logic from #359). +3. Done. No schema cloning. + +When the agent boots, it builds a **schema search path**: + +- The user's tenant org (for any custom types the user added). +- The public catalogs declared by the template's `uses_catalog` relationships. + +Vocabulary updates propagate automatically — when `public-uk-tax` adds a new +type, all agents reading it pick up the new vocabulary on next boot. Catalog +versioning is explicit at the type level (e.g. `tax_filing@2024-25` and +`tax_filing@2025-26` are separate `entity_types` rows). + +## Identity + +- One `$member` entity per (org, user). Lazy-created on first meaningful + interaction in that org, not browse. +- `entity_identities` holds technical IDs against `$member`: `auth_user_id`, + `email`, `wa_jid`, `phone`, `uk_utr`, `uk_ni`, etc. Each is + (namespace, identifier) → entity_id. +- Service agents (e.g. a public org's admin agent) get their own identities + with a `service_agent` namespace so they can be invited into private orgs + the same way human users are. + +`entity_identities` (technical lookup) and `entity_relationships` (semantic +facts) stay as separate tables. Don't conflate. + +## Contribution flow + +1. User has data in their private org they think the public catalog should + know about. +2. User notifies the public org's admin agent via the existing chat/messaging + path: *"FYI I've recorded ``."* +3. If the admin agent decides it's worth canonicalizing, it requests read + access. +4. User invites the admin agent into their private org as a `viewer` + (read-only) or `collaborator` (limited write — for the agent to stamp a + "synced as ``" reference back on the user's entity). +5. Admin's agent reads, decides, writes the canonical entity into the public + org. +6. User revokes membership when done. Or leaves the agent in for ongoing sync. + +What this gives for free: + +| Need | Reuses | +| --- | --- | +| Access mechanism | Existing org membership + role | +| Audit trail | Membership invite/revoke events | +| Revocation | DELETE membership | +| Granularity | Org boundary itself (or split a sharing sub-org for narrower control) | +| Trust UX | "Invite person/agent to org" — already familiar from Slack/Drive | + +Trade-off: invitation grants whole-org read access, not per-entity. Acceptable — +coarse and explicit beats fine-grained and hidden for trust. Users wanting +narrower control can keep contribution-bound entities in a dedicated sharing +sub-org. + +## Use case: tax return + +**`public-uk-tax`** (public_catalog) seeds: + +- HMRC, the £, tax years (2024-25, 2025-26 …), tax forms (SA100, SA102, SA105, + SA108), allowance & relief definitions, filing deadlines linked to years. + +**`public-uk-finance`** (public_catalog) seeds: + +- Major banks, large PAYE-using employers (when known), the FCA, Companies + House. + +**User's tenant org** holds: + +- `$member` with identities `auth_user_id`, `email`, `uk_utr`, `uk_ni`. +- One `tax_filing` entity per year. Relationships: `for_tax_year` → public tax + year, `filed_with` → HMRC, `taxpayer` → `$member`, `includes_form` → + form-instance entities. +- `income` entities (salary, dividends, interest), each with `source` → + bank/employer in public-uk-finance. +- `expense` entities, `allowance_claim` entities pointing at public allowance + definitions. + +The agent's job at filing time is a graph walk: from `$member` → `taxpayer` → +filing → income/expense relationships, resolving sources via cross-org +references. + +## Use case: agent community + +**`public-templates`** (public_catalog) holds one entity per published template +(type `agent_template`): + +- Forks: `forked_from` between template entities. +- Versions: either entity-per-version with `next_version` edges, or + `template_version` child entities. Both fit the graph. +- Authorship: `authored_by` from template → `$member` of the author in some + org. + +**`public-community`** (public_catalog, separate org for policy reasons): + +- `review` entities, with `reviews` → template, `authored_by` → `$member`. +- Ratings as entity properties or `rated` relationships with numeric metadata. +- Tags / categories as entities with `tagged` relationships. + +Splitting `public-templates` and `public-community` reflects different admin +policies (templates are author-editable; reviews are write-once-by-author) +without inventing new permission machinery. + +## Phase 1 — implementation scope + +1. Migration: add `kind` + `visibility` columns to `organization`. Add explicit + `source_organization_id` + `target_organization_id` columns to + `entity_relationships`. +2. App-level write guard on `entity_relationships` inserts: target must be + same-org or `visibility='public'`. +3. Seed `public-uk-tax` and `public-uk-finance` orgs with canonical UK + entities. +4. Slim install endpoint (replaces #357's mirroring): insert agent row in + tenant + provision `$member` identity. +5. Identity provisioning (keep #359). +6. Search endpoint scoped to `visibility=public` orgs. +7. Update template authoring: templates become entities of type + `agent_template` in a public catalog with `uses_catalog` relationships. + Bot phone moves from env to `agent_template` metadata. + +## Deferred (with rationale) + +| Deferred | Why now isn't the right time | Cheap to add later? | +| --- | --- | --- | +| Postgres RLS | Not required given one-directional refs + scope-local reads. App-level enforcement already in place. | Yes — separate project | +| Claims (verification status machine, evidence refs, expiry, dispute primitives, permissions table) | Not needed for tax return or initial community. Real complexity (per pi: status machine, cardinality per type, dispute states, permission projections). | Yes — additive columns + new permission table | +| Aliases / merges / tombstones for canonical entities | Needed at meaningful catalog scale. Premature now. | Yes — new tables, no existing-row migration | +| Federation (cross-instance entity references) | No multi-instance need yet. UUIDs from day one keep this option open. | Yes | +| Fine-grained per-entity sharing | Whole-org invite is coarser but explicit; serves the immediate need. | Yes — sub-orgs are the escape hatch | + +## Long-term invariants worth preserving + +1. **Vocabulary-as-data** — adding entity types or relationship types is an + INSERT, not a migration. +2. **UUIDs everywhere** — keeps federation cheap. +3. **One graph, many orgs** — orgs are trust slices through one universal + graph. +4. **Cross-org references unidirectional** (tenant → public). +5. **`entity_identities` (technical) ≠ `entity_relationships` (semantic)** — + keep them separate. + +## Implementation arc — finishing the existing work + +Status of in-flight PRs and how each lands under the new model. + +### Wave 1 — independent, ready to land now + +These don't depend on the world-model schema and aren't affected by the +template-cloning rollback. Land in any order. + +| PR | Title | Notes | +| --- | --- | --- | +| #352 | personal-org-on-signup | Creates `tenant`-kind org for new users. As-is. | +| #350 | personal-finance example | Pure content under `/agents/personal-finance/`. As-is. | +| #354 | SA100 assembly playbook | Content. As-is. | +| #355 | statement ingestion playbook | Content. As-is. | +| #356 | personal-finance evals | Content. As-is. | +| #348 | multi-org execute MCP tools | Orthogonal scaffolding. As-is. | + +### Wave 2 — world-model schema (new branches) + +Two small PRs, sequential. Total ~200 LOC including migrations + tests. + +| Branch | Scope | +| --- | --- | +| `feat/world-model-orgs` | Add `organization.kind` (`tenant | public_catalog`) + `organization.visibility` (`private | public`). Default existing rows to `tenant`/`private`. | +| `feat/world-model-relations` | Add `entity_relationships.source_organization_id` + `target_organization_id`. Backfill from current implicit scoping. App-level write guard helper rejecting cross-org targets unless target org is `public_catalog`. | + +### Wave 3 — public catalog seeding (re-targeted existing work) + +The vocabulary already in flight maps cleanly onto public catalogs. + +| PR | Re-targeting | +| --- | --- | +| #358 — company-aware world model | Re-target so the personal-finance template **org** becomes a `public_catalog` org (`kind=public_catalog`, `visibility=public`). YAML content lands as `entity_types` rows in that org. No content rewrite needed; only the seed pipeline & org metadata change. | +| #360 — phase 2 schema (FX, allowance windows, filing timeline) | Same treatment. Stacks on #358. | +| `feat/agent-template-entity` (new) | Define the `agent_template` entity type. Seed the personal-finance `agent_template` entity with metadata (system prompt, model, skill list, **bot phone**) and `uses_catalog` relationships pointing at the catalog orgs from #358/#360. | + +### Wave 4 — slim install + identity (replacing #357 / #362, keeping #359's helper) + +Three PRs, all stacked on Wave 2 + Wave 3. + +| Branch | Scope | Replaces | +| --- | --- | --- | +| `feat/slim-install` | `POST /api/install` accepts `{ template_entity_id, whatsapp_phone? }` (or slug → server-resolved to template entity). Inserts agent row in user's tenant with `template_entity_id`. Returns redirect. ~50 LOC. | #357 | +| `feat/identity-provisioning` | Salvage from #359: keep `auth/subject-identities.ts` (the helpers + signup-hook call). Drop the install-routes changes — slim-install owns those. Rebase onto `feat/slim-install`. | trims #359 | +| `feat/install-manifest-data` | `GET /api/install/manifest/:slug` reads from the `agent_template` entity. No env vars. | #362 | + +### Wave 5 — landing page + +Two-PR ship (per AGENTS.md submodule rule): + +1. owletto-web PR (the existing #20 there): update POST body to send + `template_entity_id`. Land first. +2. Parent submodule-bump PR. + +### Cleanup as PRs land + +- Close #357 when `feat/slim-install` lands. +- Close #362 when `feat/install-manifest-data` lands. +- Reduce #359 to just the salvageable helper or close + land via + `feat/identity-provisioning`. +- Delete local worktrees for `feat/install-endpoint` and + `feat/schema-mirror-install-flow` once detached. + +### Parallelism / sequencing summary + +``` +Wave 1 (any order) ─────── land independently + │ +Wave 2 ─── orgs ──── relations + │ +Wave 3 ─── #358 ── #360 ── agent-template-entity + │ +Wave 4 ─── slim-install ── identity-provisioning + │ + └── install-manifest-data + │ +Wave 5 ─── owletto-web#20 ── parent bump +``` + +Wave 1 lands now. Waves 2 and 3 can be developed in parallel by different +agents (different files), as long as 3 lands after 2's columns exist. Wave 4 +stacks on both. + +### Open questions to resolve before Wave 3 + +1. **Template org** — does personal-finance live in its own catalog org, in + `public-uk-tax`, or in a `public-templates` org? Recommendation: own org + (`public-personal-finance` or `public-templates`) for clean admin policy + separation. The `agent_template` entity references the *vocabulary* catalogs + via `uses_catalog`, so co-locating with the vocabulary isn't required. +2. **Seed mechanism** — the existing `/agents/personal-finance/` YAML + pipeline needs a small adapter so YAMLs become entity_types/entity rows in + the public catalog org. Currently they get cloned per tenant via + `installAgentFromTemplate` (now closed). Adapter is straightforward but + needs to land before #358's content can flow through. +3. **Slug → template_entity_id resolution** — keep slugs in the URL + (`/install/personal-finance`) but resolve server-side. Slug becomes a + property on the `agent_template` entity. No env-var map. diff --git a/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts b/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts index bacc7f6a8..ed2b41b11 100644 --- a/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts +++ b/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts @@ -80,8 +80,9 @@ describe('whatsapp connector > entityLinks', () => { const entitiesAfterFirst = await sql< { id: number; name: string; metadata: Record }[] >` - SELECT id, name, metadata FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT e.id, e.name, e.metadata FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entitiesAfterFirst).toHaveLength(1); expect(entitiesAfterFirst[0].name).toBe('Alex'); @@ -120,8 +121,9 @@ describe('whatsapp connector > entityLinks', () => { }); const countAfterSecond = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(countAfterSecond[0].count).toBe('1'); }); @@ -149,8 +151,9 @@ describe('whatsapp connector > entityLinks', () => { }); const count = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(count[0].count).toBe('0'); }); @@ -185,8 +188,9 @@ describe('whatsapp connector > entityLinks', () => { }); const count = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(count[0].count).toBe('0'); }); diff --git a/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts b/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts index 1d548dea1..2b5842cf6 100644 --- a/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts +++ b/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts @@ -48,11 +48,11 @@ describe('$member visibility policy on public orgs', () => { await sql` INSERT INTO entities ( - name, slug, entity_type, organization_id, metadata, created_by, created_at, updated_at + name, slug, entity_type_id, organization_id, metadata, created_by, created_at, updated_at ) VALUES ( 'Plain Member', 'plain-member', - '$member', + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${publicOrg.id} AND deleted_at IS NULL), ${publicOrg.id}, ${sql.json({ email: MEMBER_EMAIL, status: 'active', role: 'member' })}, ${adminUser.id}, diff --git a/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts b/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts index 6286a39f7..4ef4c82d7 100644 --- a/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts +++ b/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts @@ -277,11 +277,26 @@ export async function createTestEntity(options: { } } + const entityTypeSlug = options.entity_type || 'brand'; + const typeRows = await sql<{ id: number }[]>` + SELECT id FROM entity_types + WHERE slug = ${entityTypeSlug} + AND organization_id = ${options.organization_id} + AND deleted_at IS NULL + LIMIT 1 + `; + if (typeRows.length === 0) { + throw new Error( + `createTestEntity: entity_type '${entityTypeSlug}' not registered in org ${options.organization_id}` + ); + } + const entityTypeId = typeRows[0].id; + const [inserted] = await sql` INSERT INTO entities ( name, slug, - entity_type, + entity_type_id, organization_id, parent_id, metadata, @@ -291,7 +306,7 @@ export async function createTestEntity(options: { ) VALUES ( ${options.name}, ${slug}, - ${options.entity_type || 'brand'}, + ${entityTypeId}, ${options.organization_id}, ${options.parent_id || null}, ${sql.json(metadata)}, diff --git a/packages/owletto-backend/src/auth/subject-identities.ts b/packages/owletto-backend/src/auth/subject-identities.ts index 8b2942433..0deaa2bd1 100644 --- a/packages/owletto-backend/src/auth/subject-identities.ts +++ b/packages/owletto-backend/src/auth/subject-identities.ts @@ -55,11 +55,13 @@ async function findMemberEntityIdByEmail( ): Promise { const { emailField } = await resolveMemberSchemaFields(organizationId); const rows = await sql.unsafe( - `SELECT id FROM entities - WHERE entity_type = '$member' - AND organization_id = $1 - AND metadata->>$2 = $3 - AND deleted_at IS NULL + `SELECT e.id + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = '$member' + AND e.organization_id = $1 + AND e.metadata->>$2 = $3 + AND e.deleted_at IS NULL LIMIT 1`, [organizationId, emailField, email] ); diff --git a/packages/owletto-backend/src/index.ts b/packages/owletto-backend/src/index.ts index f4a6416eb..fbe0b3213 100644 --- a/packages/owletto-backend/src/index.ts +++ b/packages/owletto-backend/src/index.ts @@ -698,18 +698,19 @@ app.get('/api/:orgSlug/watchers/windows/:windowId', mcpAuth, async (c) => { i.slug as watcher_slug, i.name as watcher_name, e.name as entity_name, - e.entity_type, + et.slug AS entity_type, parent.name as parent_name, CAST(COUNT(iwf.event_id) AS INTEGER) as content_count FROM watcher_windows iw JOIN watchers i ON iw.watcher_id = i.id JOIN entities e ON e.id = ANY(i.entity_ids) + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id LEFT JOIN watcher_window_events iwf ON iwf.window_id = iw.id WHERE iw.id = ${windowId} AND e.organization_id = ${organizationId} AND i.status = 'active' - GROUP BY iw.id, i.entity_ids, i.slug, i.name, e.name, e.entity_type, parent.name + GROUP BY iw.id, i.entity_ids, i.slug, i.name, e.name, et.slug, parent.name `; if (windowResult.length === 0) { diff --git a/packages/owletto-backend/src/public-pages.ts b/packages/owletto-backend/src/public-pages.ts index 51719a007..cf8eb96a3 100644 --- a/packages/owletto-backend/src/public-pages.ts +++ b/packages/owletto-backend/src/public-pages.ts @@ -352,7 +352,7 @@ async function getPublicEntityType( SELECT COUNT(*)::int FROM entities e WHERE e.organization_id = et.organization_id - AND e.entity_type = et.slug + AND e.entity_type_id = et.id AND e.deleted_at IS NULL ) AS entity_count FROM entity_types et @@ -1026,12 +1026,13 @@ export async function buildSitemapEntries(origin: string): Promise> { const sql = getDb(); const rows = await sql` - SELECT entity_type, COUNT(*)::int as entity_count - FROM entities - WHERE organization_id = ${organizationId} - AND deleted_at IS NULL - GROUP BY entity_type + SELECT et.slug AS entity_type, COUNT(*)::int as entity_count + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + GROUP BY et.slug `; const counts = new Map(); for (const row of rows) { @@ -301,10 +302,11 @@ async function getEntityCountForType(slug: string, organizationId: string): Prom const sql = getDb(); const rows = await sql` SELECT COUNT(*)::int as count - FROM entities - WHERE entity_type = ${slug} - AND organization_id = ${organizationId} - AND deleted_at IS NULL + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = ${slug} + AND e.organization_id = ${organizationId} + AND e.deleted_at IS NULL `; return Number(rows[0]?.count || 0); } diff --git a/packages/owletto-backend/src/tools/admin/manage_watchers.ts b/packages/owletto-backend/src/tools/admin/manage_watchers.ts index a08e36aae..71565c4b7 100644 --- a/packages/owletto-backend/src/tools/admin/manage_watchers.ts +++ b/packages/owletto-backend/src/tools/admin/manage_watchers.ts @@ -902,10 +902,12 @@ async function handleCreate( const entityResult = await sql` SELECT - e.id, e.entity_type, e.parent_id, e.slug, e.organization_id, - parent.slug as parent_slug, parent.entity_type as parent_entity_type + e.id, et.slug AS entity_type, e.parent_id, e.slug, e.organization_id, + parent.slug as parent_slug, pet.slug as parent_entity_type FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE e.id = ${entityId} `; if (entityResult.length === 0) { @@ -1059,7 +1061,10 @@ async function handleCreateFromVersion( // Fetch entity names for name pattern substitution const entityRows = await sql` - SELECT id, name, entity_type, slug FROM entities WHERE id = ANY(${`{${args.entity_ids.join(',')}}`}::bigint[]) + SELECT e.id, e.name, et.slug AS entity_type, e.slug + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ANY(${`{${args.entity_ids.join(',')}}`}::bigint[]) `; const entityMap = new Map(entityRows.map((e: any) => [Number(e.id), e])); @@ -1791,7 +1796,12 @@ async function handleCompleteWindow( const eIds = Array.isArray(row.entity_ids) ? row.entity_ids.map(Number) : []; const entityRows = eIds.length > 0 - ? await sql`SELECT id, name, entity_type, metadata FROM entities WHERE id = ANY(${`{${eIds.join(',')}}`}::bigint[])` + ? await sql` + SELECT e.id, e.name, et.slug AS entity_type, e.metadata + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ANY(${`{${eIds.join(',')}}`}::bigint[]) + ` : []; // Fetch watcher name from version, slug from template (pre-consolidation) @@ -2069,14 +2079,14 @@ async function handleList( wr.created_at as watcher_run_created_at, wr.completed_at as watcher_run_completed_at, e.id as entity_id, - e.entity_type, + et.slug AS entity_type, e.name as entity_name, e.slug as entity_slug, e.organization_id, parent.id as parent_id, parent.name as parent_name, parent.slug as parent_slug, - parent.entity_type as parent_entity_type, + pet.slug as parent_entity_type, i.current_version_id, (SELECT COUNT(*) FROM watcher_windows iw WHERE iw.watcher_id = i.id) as windows_count `; @@ -2098,7 +2108,9 @@ async function handleList( query += ` FROM watchers i LEFT JOIN entities e ON e.id = ANY(i.entity_ids) + LEFT JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id LEFT JOIN watcher_versions cv ON i.current_version_id = cv.id ${buildLatestWatcherRunJoinSql('i', 'wr')} `; diff --git a/packages/owletto-backend/src/tools/get_content.ts b/packages/owletto-backend/src/tools/get_content.ts index 97e04af3d..d408088b0 100644 --- a/packages/owletto-backend/src/tools/get_content.ts +++ b/packages/owletto-backend/src/tools/get_content.ts @@ -553,14 +553,16 @@ export async function getContent( const entityResult = await sql` SELECT e.id, - e.entity_type, + et.slug AS entity_type, e.slug, e.parent_id, parent.slug as parent_slug, - parent.entity_type as parent_entity_type, + pet.slug as parent_entity_type, e.organization_id FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE e.id = ${entityId} `; @@ -1218,7 +1220,10 @@ export async function getContent( const uniqueEntityIds = Array.from(entityCountMap.keys()); const idList = `{${uniqueEntityIds.join(',')}}`; const entityRows = await sql` - SELECT id, name, entity_type FROM entities WHERE id = ANY(${idList}::int[]) + SELECT e.id, e.name, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ANY(${idList}::int[]) `; const entitySummary = entityRows @@ -1352,7 +1357,7 @@ async function handleWatcherMode( cv.condensation_prompt, cv.condensation_window_count, cv.version_sources, - (SELECT COALESCE(json_agg(json_build_object('id', e.id, 'name', e.name, 'type', e.entity_type)), '[]'::json) FROM entities e WHERE e.id = ANY(i.entity_ids)) as entities + (SELECT COALESCE(json_agg(json_build_object('id', e.id, 'name', e.name, 'type', et.slug)), '[]'::json) FROM entities e JOIN entity_types et ON et.id = e.entity_type_id WHERE e.id = ANY(i.entity_ids)) as entities FROM watchers i LEFT JOIN watcher_versions cv ON i.current_version_id = cv.id WHERE i.id = ${watcherId} diff --git a/packages/owletto-backend/src/tools/get_watchers.ts b/packages/owletto-backend/src/tools/get_watchers.ts index 52be4aacb..57bc107d1 100644 --- a/packages/owletto-backend/src/tools/get_watchers.ts +++ b/packages/owletto-backend/src/tools/get_watchers.ts @@ -319,11 +319,13 @@ export async function getWatcher( if (args.entity_id) { const entityCheck = await sql` - SELECT e.id, e.name, e.entity_type, e.slug, e.parent_id, - parent.slug as parent_slug, parent.entity_type as parent_entity_type, + SELECT e.id, e.name, et.slug AS entity_type, e.slug, e.parent_id, + parent.slug as parent_slug, pet.slug as parent_entity_type, e.organization_id FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE e.id = ${args.entity_id} `; @@ -336,12 +338,14 @@ export async function getWatcher( entitiesForTemplate = [{ name: info.entityName ?? '', type: info.entityType ?? '' }]; } else if (args.watcher_id) { const watcherEntityQuery = await sql` - SELECT e.id, e.name, e.entity_type, e.slug, e.parent_id, - parent.slug as parent_slug, parent.entity_type as parent_entity_type, + SELECT e.id, e.name, et.slug AS entity_type, e.slug, e.parent_id, + parent.slug as parent_slug, pet.slug as parent_entity_type, e.organization_id FROM watchers i JOIN entities e ON e.id = ANY(i.entity_ids) + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE i.id = ${args.watcher_id} `; @@ -1085,16 +1089,17 @@ export async function getWatcher( SELECT CAST(e.id AS TEXT) as entity_id, e.name as entity_name, - e.entity_type, + et.slug AS entity_type, COUNT(DISTINCT f.id) as total_content, COUNT(DISTINCT c.connector_key) as active_connections, MAX(f.occurred_at) as latest_content_date FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN feeds fc ON e.id = ANY(fc.entity_ids) AND fc.deleted_at IS NULL LEFT JOIN connections c ON c.id = fc.connection_id AND c.organization_id = e.organization_id LEFT JOIN current_event_records f ON ${sql.unsafe(entityLinkMatchSql('e.id::bigint', 'f'))} WHERE e.id = ${contextEntityId} - GROUP BY e.id, e.name, e.entity_type + GROUP BY e.id, e.name, et.slug `; if (entityContextQuery.length > 0) { diff --git a/packages/owletto-backend/src/tools/resolve_path.ts b/packages/owletto-backend/src/tools/resolve_path.ts index 0c44d914a..cf663b05a 100644 --- a/packages/owletto-backend/src/tools/resolve_path.ts +++ b/packages/owletto-backend/src/tools/resolve_path.ts @@ -355,10 +355,11 @@ async function _resolvePath( if (!isLeaf) { // Lightweight query for intermediate path entities – no COUNT subqueries, no template joins const row = await simpleQuery(sql` - SELECT e.id, e.entity_type, e.slug, e.name, e.parent_id + SELECT e.id, et.slug AS entity_type, e.slug, e.name, e.parent_id FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.organization_id = ${workspace.id} - AND e.entity_type = ${segment.entity_type} + AND et.slug = ${segment.entity_type} AND e.slug = ${segment.slug} AND ( (${parentId}::bigint IS NULL AND e.parent_id IS NULL) @@ -389,7 +390,7 @@ async function _resolvePath( const row = await simpleQuery(sql` SELECT e.id, - e.entity_type, + et.slug AS entity_type, e.slug, e.name, e.parent_id, @@ -398,15 +399,13 @@ async function _resolvePath( COALESCE(vtv_entity.json_template, vtv_et.json_template) as json_template, COALESCE(vtv_entity.version, vtv_et.version) as json_template_version FROM entities e - LEFT JOIN entity_types et - ON et.slug = e.entity_type - AND et.organization_id = e.organization_id + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN view_template_versions vtv_entity ON vtv_entity.id = e.current_view_template_version_id LEFT JOIN view_template_versions vtv_et ON vtv_et.id = et.current_view_template_version_id WHERE e.organization_id = ${workspace.id} - AND e.entity_type = ${segment.entity_type} + AND et.slug = ${segment.entity_type} AND e.slug = ${segment.slug} AND ( (${parentId}::bigint IS NULL AND e.parent_id IS NULL) @@ -534,18 +533,20 @@ async function _resolvePath( // content_count is omitted to avoid expensive GIN index scans over the events table. const [childRows, siblingRows] = await Promise.all([ simpleQuery(sql` - SELECT e.id, e.entity_type, e.slug, e.name, + SELECT e.id, et.slug AS entity_type, e.slug, e.name, e.metadata::jsonb->>'market' as market FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.organization_id = ${workspace.id} AND e.parent_id = ${resolvedEntity.id} ORDER BY e.name ASC `), simpleQuery(sql` - SELECT e.id, e.entity_type, e.slug, e.name + SELECT e.id, et.slug AS entity_type, e.slug, e.name FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.organization_id = ${workspace.id} - AND e.entity_type = ${resolvedEntity.entity_type} + AND et.slug = ${resolvedEntity.entity_type} AND ( (${resolvedEntity.parent_id}::bigint IS NULL AND e.parent_id IS NULL) OR e.parent_id = ${resolvedEntity.parent_id} @@ -667,8 +668,7 @@ async function listEntityTypes( COUNT(e.id)::int AS entity_count FROM entity_types et LEFT JOIN entities e - ON e.organization_id = et.organization_id - AND e.entity_type = et.slug + ON e.entity_type_id = et.id WHERE et.deleted_at IS NULL AND et.organization_id = ${organizationId} GROUP BY et.id, et.slug, et.name, et.description, et.icon, et.color @@ -911,17 +911,19 @@ async function fetchRecentWatchers( e.name AS entity_name, e.slug AS entity_slug, parent.slug AS parent_slug, - parent.entity_type AS parent_entity_type, + pet.slug AS parent_entity_type, COALESCE(wwc.windows_count, 0)::int AS windows_count FROM scoped_watchers sw LEFT JOIN LATERAL ( - SELECT entity.id, entity.entity_type, entity.name, entity.slug, entity.parent_id + SELECT entity.id, et_ent.slug AS entity_type, entity.name, entity.slug, entity.parent_id FROM entities entity + JOIN entity_types et_ent ON et_ent.id = entity.entity_type_id WHERE entity.id = ANY(sw.entity_ids) ORDER BY entity.name ASC LIMIT 1 ) e ON TRUE LEFT JOIN entities parent ON parent.id = e.parent_id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id LEFT JOIN watcher_window_counts wwc ON wwc.watcher_id = sw.id ORDER BY COALESCE(sw.updated_at, sw.created_at) DESC `); diff --git a/packages/owletto-backend/src/tools/save_content.ts b/packages/owletto-backend/src/tools/save_content.ts index b594031f5..4aa193769 100644 --- a/packages/owletto-backend/src/tools/save_content.ts +++ b/packages/owletto-backend/src/tools/save_content.ts @@ -183,11 +183,12 @@ export async function saveContent( SELECT e.id FROM entity_identities ei JOIN entities e ON e.id = ei.entity_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE ei.organization_id = ${ctx.organizationId} AND ei.namespace = 'auth_user_id' AND ei.identifier = ${authId} AND ei.deleted_at IS NULL - AND e.entity_type = '$member' + AND et.slug = '$member' AND e.deleted_at IS NULL LIMIT 1 `; @@ -202,11 +203,12 @@ export async function saveContent( SELECT e.id FROM entity_identities ei JOIN entities e ON e.id = ei.entity_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE ei.organization_id = ${ctx.organizationId} AND ei.namespace = 'email' AND ei.identifier = ${userEmail} AND ei.deleted_at IS NULL - AND e.entity_type = '$member' + AND et.slug = '$member' AND e.deleted_at IS NULL LIMIT 1 `; diff --git a/packages/owletto-backend/src/tools/search.ts b/packages/owletto-backend/src/tools/search.ts index dc2a79e26..cb0a308b7 100644 --- a/packages/owletto-backend/src/tools/search.ts +++ b/packages/owletto-backend/src/tools/search.ts @@ -428,8 +428,8 @@ async function fetchTopEntitiesByType( // ============================================ const ENTITY_SELECT_COLUMNS = ` - e.id, e.organization_id, e.name, e.entity_type, e.slug, e.metadata, e.parent_id, - pe.name as parent_name, pe.slug as parent_slug, pe.entity_type as parent_entity_type, + e.id, e.organization_id, e.name, et.slug AS entity_type, e.slug, e.metadata, e.parent_id, + pe.name as parent_name, pe.slug as parent_slug, pet.slug as parent_entity_type, COALESCE((SELECT COUNT(*) FROM current_event_records ev WHERE ${entityLinkMatchSql('e.id::bigint', 'ev')}), 0) as content_count, COALESCE(( SELECT COUNT(DISTINCT cn.connector_key) @@ -453,7 +453,9 @@ const ENTITY_SELECT_COLUMNS = ` const ENTITY_JOINS = ` FROM entities e - LEFT JOIN entities pe ON e.parent_id = pe.id`; + JOIN entity_types et ON et.id = e.entity_type_id + LEFT JOIN entities pe ON e.parent_id = pe.id + LEFT JOIN entity_types pet ON pet.id = pe.entity_type_id`; /** * Query entities by name with optional filters @@ -509,7 +511,7 @@ async function queryEntities( // Organization filter conditions.push(`e.organization_id = $${addParam(organizationId)}`); - if (args.entity_type) conditions.push(`e.entity_type = $${addParam(args.entity_type)}`); + if (args.entity_type) conditions.push(`et.slug = $${addParam(args.entity_type)}`); if (args.parent_id) conditions.push(`e.parent_id = $${addParam(args.parent_id)}`); if (args.category) conditions.push(`e.metadata::jsonb->>'category' = $${addParam(args.category)}`); @@ -640,13 +642,14 @@ async function formatEntityResult( SELECT e.id, e.name, - e.entity_type, + et.slug AS entity_type, e.metadata::jsonb->>'market' as market, COALESCE( (SELECT COUNT(*) FROM current_event_records WHERE e.id = ANY(entity_ids)), 0 ) as content_count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.parent_id = ${primaryEntity.id} ORDER BY e.created_at DESC `; diff --git a/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts b/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts index 08e9bd2db..5abf0426e 100644 --- a/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts +++ b/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts @@ -84,8 +84,9 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const entities = await sql` - SELECT id, name, metadata FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT e.id, e.name, e.metadata FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entities).toHaveLength(1); expect(entities[0].name).toBe('Alex'); @@ -107,8 +108,12 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const [{ id: entityId }] = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'Alex', 'member-seed', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'Alex', 'member-seed', '{}'::jsonb, ${user.id} + ) RETURNING id `; await sql` @@ -138,8 +143,9 @@ describe('applyEntityLinks', () => { }); const entityCount = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entityCount[0].count).toBe('1'); @@ -156,13 +162,21 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const entA = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'A', 'member-a', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'A', 'member-a', '{}'::jsonb, ${user.id} + ) RETURNING id `; const entB = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'B', 'member-b', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'B', 'member-b', '{}'::jsonb, ${user.id} + ) RETURNING id `; await sql` @@ -194,8 +208,9 @@ describe('applyEntityLinks', () => { // No new entity created, no new identifiers accreted to either side. const entities = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entities[0].count).toBe('2'); @@ -247,8 +262,12 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const [{ id: entityId }] = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'Alex', 'member-alex', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'Alex', 'member-alex', '{}'::jsonb, ${user.id} + ) RETURNING id `; await sql` diff --git a/packages/owletto-backend/src/utils/auto-linker.ts b/packages/owletto-backend/src/utils/auto-linker.ts index b6a7ec8ef..94f2d40a5 100644 --- a/packages/owletto-backend/src/utils/auto-linker.ts +++ b/packages/owletto-backend/src/utils/auto-linker.ts @@ -35,11 +35,13 @@ async function getOrgEntities(organizationId: string): Promise= ${MIN_NAME_LENGTH} - ORDER BY length(name) DESC + SELECT e.id, e.name, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + AND length(e.name) >= ${MIN_NAME_LENGTH} + ORDER BY length(e.name) DESC `; const entities = rows.map((r) => ({ diff --git a/packages/owletto-backend/src/utils/entity-link-upsert.ts b/packages/owletto-backend/src/utils/entity-link-upsert.ts index f8592820c..ca86ca472 100644 --- a/packages/owletto-backend/src/utils/entity-link-upsert.ts +++ b/packages/owletto-backend/src/utils/entity-link-upsert.ts @@ -178,10 +178,11 @@ async function lookupMatches(params: { SELECT ei.entity_id, ei.namespace, ei.identifier FROM entity_identities ei JOIN entities e ON e.id = ei.entity_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE ei.organization_id = ${params.orgId} AND ei.deleted_at IS NULL AND e.deleted_at IS NULL - AND e.entity_type = ${params.entityType} + AND et.slug = ${params.entityType} AND (ei.namespace, ei.identifier) IN ( SELECT ns, ident FROM unnest(${pgTextArray(namespaces)}::text[], ${pgTextArray(identifiers)}::text[]) AS u(ns, ident) ) @@ -211,6 +212,23 @@ async function createEntityWithIdentities(params: { const metadata: Record = {}; for (const [key, value] of params.traits) metadata[key] = value; + // Resolve entity_type slug to FK on entity_types(id). + const typeRow = await sql<{ id: number }>` + SELECT id FROM entity_types + WHERE slug = ${params.entityType} + AND organization_id = ${params.orgId} + AND deleted_at IS NULL + LIMIT 1 + `; + if (typeRow.length === 0) { + logger.warn( + { entityType: params.entityType, orgId: params.orgId }, + 'entity create failed: unknown entity type' + ); + return null; + } + const entityTypeId = typeRow[0].id; + // Try a few slug variants to defuse improbable random collisions. let entityId: number | null = null; for (let attempt = 0; attempt < 3 && entityId === null; attempt++) { @@ -218,11 +236,11 @@ async function createEntityWithIdentities(params: { try { const rows = await sql<{ id: number | string }>` INSERT INTO entities ( - organization_id, entity_type, name, slug, metadata, + organization_id, entity_type_id, name, slug, metadata, created_by, created_at, updated_at ) VALUES ( - ${params.orgId}, ${params.entityType}, ${name}, ${slug}, + ${params.orgId}, ${entityTypeId}, ${name}, ${slug}, ${sql.json(metadata)}, ${params.creatorUserId}, current_timestamp, current_timestamp ) diff --git a/packages/owletto-backend/src/utils/entity-management.ts b/packages/owletto-backend/src/utils/entity-management.ts index f2a1f0c1b..c2e06dade 100644 --- a/packages/owletto-backend/src/utils/entity-management.ts +++ b/packages/owletto-backend/src/utils/entity-management.ts @@ -232,21 +232,21 @@ export async function createEntity( const sql = getDb(); - // Validate entity type exists in entity_types table - const typeCheck = await sql.unsafe( - `SELECT id FROM entity_types - WHERE slug = $1 - AND deleted_at IS NULL - AND organization_id = $2 - LIMIT 1`, - [data.entity_type, data.organization_id] - ); - if (typeCheck.length === 0) { + // Resolve entity_type slug to FK on entity_types(id). + const typeRow = await sql<{ id: number }>` + SELECT id FROM entity_types + WHERE slug = ${data.entity_type} + AND deleted_at IS NULL + AND organization_id = ${data.organization_id} + LIMIT 1 + `; + if (typeRow.length === 0) { throw new ToolUserError( `Unknown entity type '${data.entity_type}'. Use manage_entity_schema(schema_type="entity_type", action="list") to list available types or create a custom type first.`, 400 ); } + const entityTypeId = typeRow[0].id; // Generate slug from name if not provided const slug = data.slug || generateSlug(data.name); @@ -265,22 +265,24 @@ export async function createEntity( const contentHash = data.content_hash || null; try { - const result = await sql` + const result = await sql>` INSERT INTO entities ( - organization_id, entity_type, name, slug, parent_id, metadata, enabled_classifiers, created_by, content, embedding, content_hash, created_at, updated_at + organization_id, entity_type_id, name, slug, parent_id, metadata, enabled_classifiers, created_by, content, embedding, content_hash, created_at, updated_at ) VALUES ( - ${data.organization_id}, ${data.entity_type}, ${data.name.trim()}, ${slug}, ${data.parent_id || null}, + ${data.organization_id}, ${entityTypeId}, ${data.name.trim()}, ${slug}, ${data.parent_id || null}, ${sql.json(metadata)}, ${data.enabled_classifiers || null}, ${createdBy}, ${contentValue}, ${embeddingLiteral}::vector, ${contentHash}, current_timestamp, current_timestamp ) - RETURNING id, entity_type, name, slug, parent_id, metadata, created_at + RETURNING id, name, slug, parent_id, metadata, created_at `; if (result.length === 0) { throw new Error('Failed to create entity'); } - const created = result[0]; + // The validator above already resolved data.entity_type → entityTypeId. + // Pass the slug back through directly rather than JOIN-ing on every insert. + const created: CreatedEntity = { ...result[0], entity_type: data.entity_type }; // Run afterCreate hook if (!opts?.skipHooks && opts?.hookContext) { @@ -376,9 +378,10 @@ export async function updateEntity( `; const result = await sql` - SELECT id, entity_type, name, slug, parent_id, metadata, created_at - FROM entities - WHERE id = ${entityId} + SELECT e.id, et.slug AS entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ${entityId} LIMIT 1 `; @@ -403,9 +406,9 @@ export async function getEntity( const result = await sql` SELECT - e.id, e.entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, + e.id, et.slug AS entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, e.current_view_template_version_id, - pe.name as parent_name, pe.slug as parent_slug, pe.entity_type as parent_entity_type, + pe.name as parent_name, pe.slug as parent_slug, pet.slug as parent_entity_type, (SELECT COUNT(*) FROM current_event_records ev WHERE ${sql.unsafe(entityLinkMatchSql('e.id::bigint', 'ev'))}) as total_content, ( SELECT COUNT(DISTINCT c.connector_key) @@ -418,7 +421,9 @@ export async function getEntity( (SELECT COUNT(*) FROM watchers i WHERE e.id = ANY(i.entity_ids)) as watchers_count, (SELECT COUNT(*) FROM entities c WHERE c.parent_id = e.id) as children_count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities pe ON e.parent_id = pe.id + LEFT JOIN entity_types pet ON pet.id = pe.entity_type_id WHERE e.id = ${entityId} AND e.organization_id = ${ctx.organizationId} AND e.deleted_at IS NULL @@ -448,7 +453,10 @@ export async function deleteEntity( // Run beforeDelete hook if (!opts?.skipHooks) { const entityRow = await sql` - SELECT entity_type, metadata FROM entities WHERE id = ${entityId} AND deleted_at IS NULL + SELECT et.slug AS entity_type, e.metadata + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ${entityId} AND e.deleted_at IS NULL `; if (entityRow.length > 0) { const hooks = getEntityHooks(entityRow[0].entity_type as string); @@ -655,7 +663,7 @@ export async function listEntities( params.push(ctx.organizationId); if (filters.entity_type) { - conditions.push(`e.entity_type = $${paramIdx++}`); + conditions.push(`et.slug = $${paramIdx++}`); params.push(filters.entity_type); } @@ -707,7 +715,9 @@ export async function listEntities( const baseQuery = ` FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities pe ON e.parent_id = pe.id + LEFT JOIN entity_types pet ON pet.id = pe.entity_type_id LEFT JOIN LATERAL (SELECT COUNT(*) as cnt FROM current_event_records ev WHERE ${entityLinkMatchSql('e.id::bigint', 'ev')}) tc ON true LEFT JOIN LATERAL ( SELECT COUNT(DISTINCT c.connector_key) as cnt @@ -729,12 +739,12 @@ export async function listEntities( const result = await sql.unsafe( `SELECT - e.id, e.entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, + e.id, et.slug AS entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, COALESCE(tc.cnt, 0) as total_content, COALESCE(ac.cnt, 0) as active_connections, COALESCE(ic.cnt, 0) as watchers_count, COALESCE(cc.cnt, 0) as children_count, - pe.name as parent_name, pe.slug as parent_slug, pe.entity_type as parent_entity_type + pe.name as parent_name, pe.slug as parent_slug, pet.slug as parent_entity_type ${baseQuery} ORDER BY ${orderBy} LIMIT ${limit + 1} @@ -786,12 +796,14 @@ export async function batchLoadRelationships( r.from_entity_id, r.to_entity_id, rt.slug AS relationship_type_slug, - fe.id AS from_id, fe.name AS from_name, fe.slug AS from_slug, fe.entity_type AS from_entity_type, - te.id AS to_id, te.name AS to_name, te.slug AS to_slug, te.entity_type AS to_entity_type + fe.id AS from_id, fe.name AS from_name, fe.slug AS from_slug, fet.slug AS from_entity_type, + te.id AS to_id, te.name AS to_name, te.slug AS to_slug, tet.slug AS to_entity_type FROM entity_relationships r JOIN entity_relationship_types rt ON r.relationship_type_id = rt.id LEFT JOIN entities fe ON r.from_entity_id = fe.id + LEFT JOIN entity_types fet ON fet.id = fe.entity_type_id LEFT JOIN entities te ON r.to_entity_id = te.id + LEFT JOIN entity_types tet ON tet.id = te.entity_type_id WHERE r.organization_id = ${organizationId} AND r.deleted_at IS NULL AND rt.slug = ANY(${typeSlugs}::text[]) diff --git a/packages/owletto-backend/src/utils/event-kind-validation.ts b/packages/owletto-backend/src/utils/event-kind-validation.ts index d336be342..5492fc3b8 100644 --- a/packages/owletto-backend/src/utils/event-kind-validation.ts +++ b/packages/owletto-backend/src/utils/event-kind-validation.ts @@ -134,7 +134,7 @@ async function getEntityTypeEventKinds( const rows = await sql` SELECT et.event_kinds FROM entities e - JOIN entity_types et ON et.slug = e.entity_type AND et.organization_id = e.organization_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.id = ${entityId} AND e.organization_id = ${orgId} AND e.deleted_at IS NULL diff --git a/packages/owletto-backend/src/utils/execute-data-sources.ts b/packages/owletto-backend/src/utils/execute-data-sources.ts index 71c18dfa9..0f13c6386 100644 --- a/packages/owletto-backend/src/utils/execute-data-sources.ts +++ b/packages/owletto-backend/src/utils/execute-data-sources.ts @@ -215,13 +215,33 @@ function buildScopedQuery( return buildColumnList(defs, alias); }; + // Build the SELECT list for the entities CTE, where entity_type is now a + // derived column from a JOIN to entity_types (et.slug AS entity_type). + const selEntitiesJoined = (entityAlias: string, typeAlias: string): string => { + const defs = sc?.get('entities'); + if (!defs) return `${entityAlias}.*, ${typeAlias}.slug AS entity_type`; + return defs + .map((c) => { + if (c.name === 'entity_type') return `${typeAlias}.slug AS "entity_type"`; + if (c.expr) { + const prefixed = c.expr.replace(/^(\w+)/, `${entityAlias}.$1`); + return `${prefixed} as "${c.name}"`; + } + return `${entityAlias}."${c.name}"`; + }) + .join(', '); + }; + for (const table of tableRefs) { // Escape double quotes in table name for safe identifier quoting const safeName = table.replace(/"/g, '""'); if (table === 'entities') { ctes.push( - `"${safeName}" AS (SELECT ${sel(table)} FROM public.entities WHERE organization_id = ${orgP})` + `"${safeName}" AS (SELECT ${selEntitiesJoined('e', 'et')} ` + + `FROM public.entities e ` + + `JOIN public.entity_types et ON et.id = e.entity_type_id ` + + `WHERE e.organization_id = ${orgP})` ); } else if (table === 'events') { let eventsCte = @@ -318,7 +338,10 @@ function buildScopedQuery( idx++; params.push(table); ctes.push( - `"${safeName}" AS (SELECT ${sel('entities')} FROM public.entities WHERE organization_id = ${orgP} AND entity_type = $${idx})` + `"${safeName}" AS (SELECT ${selEntitiesJoined('e', 'et')} ` + + `FROM public.entities e ` + + `JOIN public.entity_types et ON et.id = e.entity_type_id ` + + `WHERE e.organization_id = ${orgP} AND et.slug = $${idx})` ); } } diff --git a/packages/owletto-backend/src/utils/member-entity.ts b/packages/owletto-backend/src/utils/member-entity.ts index ef5ba2ddd..c1cfba7c2 100644 --- a/packages/owletto-backend/src/utils/member-entity.ts +++ b/packages/owletto-backend/src/utils/member-entity.ts @@ -52,11 +52,13 @@ export async function ensureMemberEntity(params: EnsureMemberEntityParams): Prom // Check if a $member entity with this email already exists const existing = await sql.unsafe( - `SELECT id FROM entities - WHERE entity_type = '$member' - AND organization_id = $1 - AND metadata->>$2 = $3 - AND deleted_at IS NULL + `SELECT e.id + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = '$member' + AND e.organization_id = $1 + AND e.metadata->>$2 = $3 + AND e.deleted_at IS NULL LIMIT 1`, [params.organizationId, emailField, params.email] ); @@ -95,7 +97,10 @@ export async function updateMemberEntityStatus( UPDATE entities SET metadata = jsonb_set(metadata, '{status}', to_jsonb(${status}::text)), updated_at = current_timestamp - WHERE entity_type = '$member' + WHERE entity_type_id = ( + SELECT id FROM entity_types + WHERE slug = '$member' AND organization_id = ${organizationId} AND deleted_at IS NULL + ) AND organization_id = ${organizationId} AND metadata->>${emailField} = ${email} AND deleted_at IS NULL @@ -111,11 +116,13 @@ export async function updateMemberEntityAccess( const { emailField } = await resolveMemberSchemaFields(organizationId); const sql = getDb(); const rows = await sql.unsafe<{ id: number; metadata: Record }>( - `SELECT id, metadata FROM entities - WHERE entity_type = '$member' - AND organization_id = $1 - AND metadata->>$2 = $3 - AND deleted_at IS NULL + `SELECT e.id, e.metadata + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = '$member' + AND e.organization_id = $1 + AND e.metadata->>$2 = $3 + AND e.deleted_at IS NULL LIMIT 1`, [organizationId, emailField, email] ); @@ -143,7 +150,10 @@ export async function deleteMemberEntity(organizationId: string, email: string): await sql.unsafe( `UPDATE entities SET deleted_at = current_timestamp, updated_at = current_timestamp - WHERE entity_type = '$member' + WHERE entity_type_id = ( + SELECT id FROM entity_types + WHERE slug = '$member' AND organization_id = $1 AND deleted_at IS NULL + ) AND organization_id = $1 AND metadata->>$2 = $3 AND deleted_at IS NULL`, diff --git a/packages/owletto-backend/src/utils/relationship-validation.ts b/packages/owletto-backend/src/utils/relationship-validation.ts index db53479ec..ac2fbe1ba 100644 --- a/packages/owletto-backend/src/utils/relationship-validation.ts +++ b/packages/owletto-backend/src/utils/relationship-validation.ts @@ -115,9 +115,10 @@ export async function validateTypeRule( // Get entity types for both entities const entityRows = await sql` - SELECT id, entity_type - FROM entities - WHERE id IN (${fromEntityId}, ${toEntityId}) + SELECT e.id, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id IN (${fromEntityId}, ${toEntityId}) `; const fromEntityType = String(entityRows.find((r) => Number(r.id) === fromEntityId)?.entity_type); const toEntityType = String(entityRows.find((r) => Number(r.id) === toEntityId)?.entity_type); diff --git a/packages/owletto-backend/src/utils/table-schema.ts b/packages/owletto-backend/src/utils/table-schema.ts index bccac23e3..49f1840be 100644 --- a/packages/owletto-backend/src/utils/table-schema.ts +++ b/packages/owletto-backend/src/utils/table-schema.ts @@ -26,6 +26,8 @@ function cols(...names: string[]): ColumnDef[] { export const QUERYABLE_SCHEMA = { tables: [ // entities (excludes: embedding, content_tsv, content_hash) + // entity_type is exposed as a derived column — the CTE JOINs entity_types + // and aliases et.slug AS entity_type, so user queries can keep referencing it. { name: 'entities', columns: cols( From 01c23330be7d26b32bbf25f9eb71000757b24b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sun, 26 Apr 2026 18:40:34 +0100 Subject: [PATCH 2/5] =?UTF-8?q?fix(entity-type-fk):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20missed=20SQL=20sites=20+=20deterministic=20backfill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pi review caught three SELECTs that referenced the (now-dropped) entity_type text column and would crash post-migration: - tools/admin/manage_entity.ts:545 — handleUpdate fetched before-state - tools/search.ts:405 — fetchTopEntitiesByType - utils/workspace-instructions.ts:25 — entity-type counts Each rewritten to JOIN entity_types and select et.slug AS entity_type. Backfill in the migration was non-deterministic: the (slug, organization_id) pair has multiple matches when an active entity_types row coexists with a soft-deleted predecessor, since the UNIQUE index only covers `deleted_at IS NULL`. Switched to a correlated subquery ordered by `(deleted_at IS NULL) DESC, id DESC` so live rows always win, soft-deleted rows are the fallback for history preservation. --- ...20260426120000_entities_entity_type_fk.sql | 19 +++++++++++++------ .../src/tools/admin/manage_entity.ts | 7 ++++--- packages/owletto-backend/src/tools/search.ts | 11 ++++++----- .../src/utils/workspace-instructions.ts | 10 ++++++---- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/db/migrations/20260426120000_entities_entity_type_fk.sql b/db/migrations/20260426120000_entities_entity_type_fk.sql index dc3a5216f..f5f39b0e2 100644 --- a/db/migrations/20260426120000_entities_entity_type_fk.sql +++ b/db/migrations/20260426120000_entities_entity_type_fk.sql @@ -22,13 +22,20 @@ ALTER TABLE public.entities ADD COLUMN entity_type_id integer REFERENCES public.entity_types(id); -- 2. Backfill from existing (organization_id, entity_type slug) → entity_types.id. --- Soft-deleted entity_types still resolve — preserves history of soft-removed types. +-- Prefer live entity_types rows; fall back to soft-deleted ones to preserve +-- history. Without the ORDER BY, a slug+org pair with both an active and a +-- soft-deleted row would resolve non-deterministically — entity_types' UNIQUE +-- index on slug only covers `deleted_at IS NULL` rows, so collisions can exist. UPDATE public.entities e -SET entity_type_id = et.id -FROM public.entity_types et -WHERE et.slug = e.entity_type - AND et.organization_id = e.organization_id - AND e.entity_type_id IS NULL; +SET entity_type_id = ( + SELECT et.id + FROM public.entity_types et + WHERE et.slug = e.entity_type + AND et.organization_id = e.organization_id + ORDER BY (et.deleted_at IS NULL) DESC, et.id DESC + LIMIT 1 +) +WHERE e.entity_type_id IS NULL; -- 3. Fail loudly on orphans. If any entities reference a slug with no matching -- entity_types row, that's pre-existing data corruption from the slug-based diff --git a/packages/owletto-backend/src/tools/admin/manage_entity.ts b/packages/owletto-backend/src/tools/admin/manage_entity.ts index 8139eff8b..d3e3b76fe 100644 --- a/packages/owletto-backend/src/tools/admin/manage_entity.ts +++ b/packages/owletto-backend/src/tools/admin/manage_entity.ts @@ -542,9 +542,10 @@ async function handleUpdate( // Fetch before state for change tracking and validation const beforeRows = await sql` - SELECT name, slug, parent_id, metadata, entity_type - FROM entities - WHERE id = ${entityId} AND deleted_at IS NULL + SELECT e.name, e.slug, e.parent_id, e.metadata, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ${entityId} AND e.deleted_at IS NULL `; if (beforeRows.length === 0) { throw new Error(`Entity with ID ${entityId} not found`); diff --git a/packages/owletto-backend/src/tools/search.ts b/packages/owletto-backend/src/tools/search.ts index cb0a308b7..1517e4f21 100644 --- a/packages/owletto-backend/src/tools/search.ts +++ b/packages/owletto-backend/src/tools/search.ts @@ -402,11 +402,12 @@ async function fetchTopEntitiesByType( ): Promise }>> { const sql = getDb(); const rows = await sql` - SELECT id, name, entity_type - FROM entities - WHERE organization_id = ${organizationId} - AND deleted_at IS NULL - ORDER BY (SELECT COUNT(*) FROM current_event_records ev WHERE ${sql.unsafe(entityLinkMatchSql('entities.id::bigint', 'ev'))}) DESC + SELECT e.id, e.name, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + ORDER BY (SELECT COUNT(*) FROM current_event_records ev WHERE ${sql.unsafe(entityLinkMatchSql('e.id::bigint', 'ev'))}) DESC LIMIT 30 `; diff --git a/packages/owletto-backend/src/utils/workspace-instructions.ts b/packages/owletto-backend/src/utils/workspace-instructions.ts index 6005f3a75..1595e7605 100644 --- a/packages/owletto-backend/src/utils/workspace-instructions.ts +++ b/packages/owletto-backend/src/utils/workspace-instructions.ts @@ -22,10 +22,12 @@ export async function buildWorkspaceInstructions(organizationId: string): Promis [organizationId] ), sql` - SELECT entity_type, COUNT(*)::int as entity_count - FROM entities - WHERE organization_id = ${organizationId} - GROUP BY entity_type + SELECT et.slug AS entity_type, COUNT(*)::int as entity_count + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + GROUP BY et.slug `, sql.unsafe( `SELECT rt.slug, rt.name, rt.is_symmetric, inv.slug as inverse_type_slug, From 82cc69cc5f517a81317c4f3f900cbbbbc84be6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sun, 26 Apr 2026 18:50:12 +0100 Subject: [PATCH 3/5] fix(migration): drop redundant unique constraint explicitly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DROP COLUMN entity_type would silently cascade-drop entities_organization_id_entity_type_slug_parent_id_key. Make the drop explicit, and document why we don't recreate it: the stronger entities_slug_parent_unique (UNIQUE on org_id, COALESCE(parent_id, 0), slug) already enforces slug uniqueness within (org, parent) regardless of entity type, with NULL-parent collapsing — so the entity-type-keyed constraint never caught anything the index didn't already catch. Down migration recreates the constraint and the column comment for rollback fidelity. Caught by pi review v2. --- ...20260426120000_entities_entity_type_fk.sql | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/db/migrations/20260426120000_entities_entity_type_fk.sql b/db/migrations/20260426120000_entities_entity_type_fk.sql index f5f39b0e2..264fc56ce 100644 --- a/db/migrations/20260426120000_entities_entity_type_fk.sql +++ b/db/migrations/20260426120000_entities_entity_type_fk.sql @@ -61,7 +61,20 @@ CREATE INDEX idx_entities_entity_type_id ON public.entities (entity_type_id) WHERE deleted_at IS NULL; --- 6. Drop the text column. All readers JOIN to entity_types for the slug. +-- 6. Drop the redundant UNIQUE constraint that referenced entity_type. The +-- stronger `entities_slug_parent_unique` (UNIQUE on org_id, COALESCE(parent_id, +-- 0), slug) already enforces slug uniqueness within (org, parent) regardless +-- of entity type, with NULL-parent collapsing — so this constraint never +-- caught anything the index didn't already catch. Drop it explicitly rather +-- than letting DROP COLUMN cascade silently. +ALTER TABLE public.entities + DROP CONSTRAINT IF EXISTS entities_organization_id_entity_type_slug_parent_id_key; + +-- 7. Drop the column comment so DROP COLUMN doesn't carry a stale doc string +-- if this migration is ever rolled back and re-applied. +COMMENT ON COLUMN public.entities.entity_type IS NULL; + +-- 8. Drop the text column. All readers JOIN to entity_types for the slug. ALTER TABLE public.entities DROP COLUMN entity_type; @@ -76,6 +89,13 @@ WHERE et.id = e.entity_type_id; ALTER TABLE public.entities ALTER COLUMN entity_type SET NOT NULL; +COMMENT ON COLUMN public.entities.entity_type IS + 'Type of entity: brand, product (future: location, feature, team)'; + +ALTER TABLE public.entities + ADD CONSTRAINT entities_organization_id_entity_type_slug_parent_id_key + UNIQUE (organization_id, entity_type, slug, parent_id); + DROP INDEX IF EXISTS public.idx_entities_entity_type_id; ALTER TABLE public.entities DROP COLUMN entity_type_id; From 17905e1ebf18c1f80413b4d4ad4f3a68957e5a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sun, 26 Apr 2026 18:54:30 +0100 Subject: [PATCH 4/5] fix(test-fixtures): auto-seed entity_type for createTestEntity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests routinely create entities in fresh orgs (e.g. createTestOrganization followed by createTestEntity) without first calling seedSystemEntityTypes. Pre-FK that worked because the slug-text column accepted any string; the new FK requires an entity_types row to exist. Either seed in every test setup or upsert here — the latter is one-time-cheap and keeps existing test code working. Caught by pi review v3. --- .../src/__tests__/setup/test-fixtures.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts b/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts index 4ef4c82d7..4a5d577d7 100644 --- a/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts +++ b/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts @@ -277,8 +277,11 @@ export async function createTestEntity(options: { } } + // Tests routinely create entities in fresh orgs without first calling + // seedSystemEntityTypes(); ensure the requested type exists so the FK + // (entities.entity_type_id) resolves without forcing every test to seed. const entityTypeSlug = options.entity_type || 'brand'; - const typeRows = await sql<{ id: number }[]>` + let typeRows = await sql<{ id: number }[]>` SELECT id FROM entity_types WHERE slug = ${entityTypeSlug} AND organization_id = ${options.organization_id} @@ -286,9 +289,11 @@ export async function createTestEntity(options: { LIMIT 1 `; if (typeRows.length === 0) { - throw new Error( - `createTestEntity: entity_type '${entityTypeSlug}' not registered in org ${options.organization_id}` - ); + typeRows = await sql<{ id: number }[]>` + INSERT INTO entity_types (organization_id, slug, name, created_at, updated_at) + VALUES (${options.organization_id}, ${entityTypeSlug}, ${entityTypeSlug}, current_timestamp, current_timestamp) + RETURNING id + `; } const entityTypeId = typeRows[0].id; From 72e6e8600ab8b54a40e265a427bee737f9dae721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sun, 26 Apr 2026 20:56:37 +0100 Subject: [PATCH 5/5] chore: regenerate db/schema.sql post-migration dbmate auto-regenerated after applying 20260426120000_entities_entity_type_fk.sql to prod. Reflects: - entities.entity_type column dropped - entities.entity_type_id integer NOT NULL FK on entity_types(id) - idx_entities_entity_type_id index - entities_organization_id_entity_type_slug_parent_id_key constraint dropped (redundant with entities_slug_parent_unique) --- db/schema.sql | 440 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 394 insertions(+), 46 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index e7bfe8f48..b7ec1b525 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,4 +1,4 @@ -\restrict yPseyczHShSrIqawjVUmfWrB9Pz35TAROpf9iU3KThZVhO4FOYhotkUDqwmder9 +\restrict bqBE913aGPDwzJsfFpJo9ktouNunyv25fIQIyfTS9VJm4SUnYw8b0K9sqZozUWV -- Dumped from database version 18.1 (Debian 18.1-1.pgdg13+2) -- Dumped by pg_dump version 18.1 (Homebrew) @@ -531,6 +531,18 @@ CREATE TABLE pgboss.warning ( ); +-- +-- Name: _reactions_backup_2026_04_25; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public._reactions_backup_2026_04_25 ( + id text NOT NULL, + reaction_script text, + reaction_script_compiled text, + backed_up_at timestamp with time zone DEFAULT now() NOT NULL +); + + -- -- Name: account; Type: TABLE; Schema: public; Owner: - -- @@ -612,6 +624,26 @@ ALTER TABLE public.agent_grants ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY ); +-- +-- Name: agent_secrets; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.agent_secrets ( + name text NOT NULL, + ciphertext text NOT NULL, + expires_at timestamp with time zone, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: TABLE agent_secrets; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.agent_secrets IS 'Encrypted secret values referenced via secret:// refs. Backs the PostgresSecretStore implementation of @lobu/gateway WritableSecretStore.'; + + -- -- Name: agent_users; Type: TABLE; Schema: public; Owner: - -- @@ -682,7 +714,8 @@ CREATE TABLE public.auth_profiles ( created_by text, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT auth_profiles_profile_kind_check CHECK ((profile_kind = ANY (ARRAY['env'::text, 'oauth_app'::text, 'oauth_account'::text, 'browser_session'::text]))), + metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + CONSTRAINT auth_profiles_profile_kind_check CHECK ((profile_kind = ANY (ARRAY['env'::text, 'oauth_app'::text, 'oauth_account'::text, 'browser_session'::text, 'interactive'::text]))), CONSTRAINT auth_profiles_status_check CHECK ((status = ANY (ARRAY['active'::text, 'pending_auth'::text, 'error'::text, 'revoked'::text]))) ); @@ -812,10 +845,19 @@ CREATE TABLE public.connector_definitions ( api_type text DEFAULT 'api'::text NOT NULL, favicon_domain text, openapi_config jsonb, + default_connection_config jsonb, + entity_link_overrides jsonb, CONSTRAINT connector_definitions_status_check CHECK ((status = ANY (ARRAY['active'::text, 'archived'::text, 'draft'::text]))) ); +-- +-- Name: COLUMN connector_definitions.entity_link_overrides; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.connector_definitions.entity_link_overrides IS 'Per-install override of connector entityLinks rules. See resolveEntityLinkRules() for merge semantics.'; + + -- -- Name: connector_definitions_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -1007,7 +1049,6 @@ CREATE VIEW public.current_event_records AS CREATE TABLE public.entities ( id bigint NOT NULL, - entity_type text NOT NULL, parent_id bigint, name text NOT NULL, metadata jsonb DEFAULT '{}'::jsonb, @@ -1022,7 +1063,8 @@ CREATE TABLE public.entities ( embedding public.vector(768), content_tsv tsvector GENERATED ALWAYS AS (to_tsvector('english'::regconfig, ((COALESCE(name, ''::text) || ' '::text) || COALESCE(content, ''::text)))) STORED, content_hash text, - deleted_at timestamp with time zone + deleted_at timestamp with time zone, + entity_type_id integer NOT NULL ); @@ -1033,13 +1075,6 @@ CREATE TABLE public.entities ( COMMENT ON TABLE public.entities IS 'Unified entity table (brands, products, and future entity types)'; --- --- Name: COLUMN entities.entity_type; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.entities.entity_type IS 'Type of entity: brand, product (future: location, feature, team)'; - - -- -- Name: COLUMN entities.parent_id; Type: COMMENT; Schema: public; Owner: - -- @@ -1073,6 +1108,70 @@ CREATE SEQUENCE public.entities_id_seq ALTER SEQUENCE public.entities_id_seq OWNED BY public.entities.id; +-- +-- Name: entity_identities; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.entity_identities ( + id bigint NOT NULL, + organization_id text NOT NULL, + entity_id bigint NOT NULL, + namespace text NOT NULL, + identifier text NOT NULL, + source_connector text, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + deleted_at timestamp with time zone +); + + +-- +-- Name: TABLE entity_identities; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.entity_identities IS 'Normalized identifier claims per entity. See docs/identity-linking.md for the full pattern.'; + + +-- +-- Name: COLUMN entity_identities.namespace; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.entity_identities.namespace IS 'Identifier kind. Standard values: phone, email, wa_jid, slack_user_id, github_login, auth_user_id, google_contact_id. Custom namespaces allowed but connectors sharing a namespace must agree on its format.'; + + +-- +-- Name: COLUMN entity_identities.identifier; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.entity_identities.identifier IS 'Normalized identifier value (E.164 digits for phone, lowercase for email, etc.). Normalizers in @lobu/owletto-sdk own the canonical form.'; + + +-- +-- Name: COLUMN entity_identities.source_connector; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.entity_identities.source_connector IS 'Who claimed this identifier: "connector:whatsapp", "manual", or null when seeded by migration.'; + + +-- +-- Name: entity_identities_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.entity_identities_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: entity_identities_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.entity_identities_id_seq OWNED BY public.entity_identities.id; + + -- -- Name: entity_relationship_type_rules; Type: TABLE; Schema: public; Owner: - -- @@ -1126,6 +1225,8 @@ CREATE TABLE public.entity_relationship_types ( deleted_at timestamp with time zone, created_at timestamp with time zone DEFAULT now(), updated_at timestamp with time zone DEFAULT now(), + managed_by_template_agent_id text, + source_template_org_id text, CONSTRAINT entity_relationship_types_status_check CHECK ((status = ANY (ARRAY['active'::text, 'archived'::text]))) ); @@ -1246,7 +1347,9 @@ CREATE TABLE public.entity_types ( created_at timestamp with time zone DEFAULT now(), updated_at timestamp with time zone DEFAULT now(), current_view_template_version_id integer, - event_kinds jsonb + event_kinds jsonb, + managed_by_template_agent_id text, + source_template_org_id text ); @@ -1379,6 +1482,8 @@ CREATE TABLE public.event_classifiers ( watcher_id bigint, organization_id text, entity_ids bigint[], + managed_by_template_agent_id text, + source_template_org_id text, CONSTRAINT event_classifiers_status_check CHECK ((status = ANY (ARRAY['active'::text, 'deprecated'::text]))) ); @@ -1870,8 +1975,12 @@ CREATE TABLE public.runs ( watcher_id integer, window_id bigint, approved_input jsonb, + auth_profile_id bigint, + auth_signal jsonb, + created_by_user_id text, + dispatched_message_id text, CONSTRAINT runs_approval_status_check CHECK ((approval_status = ANY (ARRAY['pending'::text, 'approved'::text, 'rejected'::text, 'auto'::text]))), - CONSTRAINT runs_run_type_check CHECK ((run_type = ANY (ARRAY['sync'::text, 'action'::text, 'code'::text, 'insight'::text, 'embed_backfill'::text, 'watcher'::text]))), + CONSTRAINT runs_run_type_check CHECK ((run_type = ANY (ARRAY['sync'::text, 'action'::text, 'code'::text, 'insight'::text, 'watcher'::text, 'embed_backfill'::text, 'auth'::text]))), CONSTRAINT runs_status_check CHECK ((status = ANY (ARRAY['pending'::text, 'claimed'::text, 'running'::text, 'completed'::text, 'failed'::text, 'cancelled'::text, 'timeout'::text]))) ); @@ -2268,26 +2377,29 @@ ALTER SEQUENCE public.watcher_window_content_id_seq OWNED BY public.watcher_wind -- --- Name: watcher_window_feedback; Type: TABLE; Schema: public; Owner: - +-- Name: watcher_window_field_feedback; Type: TABLE; Schema: public; Owner: - -- -CREATE TABLE public.watcher_window_feedback ( +CREATE TABLE public.watcher_window_field_feedback ( id bigint NOT NULL, window_id integer NOT NULL, watcher_id integer NOT NULL, organization_id text NOT NULL, - corrections jsonb NOT NULL, - notes text, + field_path text NOT NULL, + mutation text DEFAULT 'set'::text NOT NULL, + corrected_value jsonb, + note text, created_by text NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT watcher_window_field_feedback_mutation_check CHECK ((mutation = ANY (ARRAY['set'::text, 'remove'::text, 'add'::text]))) ); -- --- Name: watcher_window_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- Name: watcher_window_field_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- -CREATE SEQUENCE public.watcher_window_feedback_id_seq +CREATE SEQUENCE public.watcher_window_field_feedback_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -2296,10 +2408,10 @@ CREATE SEQUENCE public.watcher_window_feedback_id_seq -- --- Name: watcher_window_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- Name: watcher_window_field_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - -- -ALTER SEQUENCE public.watcher_window_feedback_id_seq OWNED BY public.watcher_window_feedback.id; +ALTER SEQUENCE public.watcher_window_field_feedback_id_seq OWNED BY public.watcher_window_field_feedback.id; -- @@ -2323,7 +2435,8 @@ CREATE TABLE public.watcher_windows ( version_id integer, depth integer DEFAULT 0, client_id text, - run_metadata jsonb DEFAULT '{}'::jsonb NOT NULL + run_metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + run_id bigint ); @@ -2400,6 +2513,8 @@ CREATE TABLE public.watchers ( scheduler_client_id text, source_watcher_id integer, watcher_group_id integer NOT NULL, + managed_by_template_agent_id text, + source_template_org_id text, CONSTRAINT insights_status_check CHECK ((status = ANY (ARRAY['active'::text, 'archived'::text]))) ); @@ -2586,6 +2701,13 @@ ALTER TABLE ONLY public.connector_versions ALTER COLUMN id SET DEFAULT nextval(' ALTER TABLE ONLY public.entities ALTER COLUMN id SET DEFAULT nextval('public.entities_id_seq'::regclass); +-- +-- Name: entity_identities id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entity_identities ALTER COLUMN id SET DEFAULT nextval('public.entity_identities_id_seq'::regclass); + + -- -- Name: entity_relationship_type_rules id; Type: DEFAULT; Schema: public; Owner: - -- @@ -2720,10 +2842,10 @@ ALTER TABLE ONLY public.watcher_window_events ALTER COLUMN id SET DEFAULT nextva -- --- Name: watcher_window_feedback id; Type: DEFAULT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback id; Type: DEFAULT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback ALTER COLUMN id SET DEFAULT nextval('public.watcher_window_feedback_id_seq'::regclass); +ALTER TABLE ONLY public.watcher_window_field_feedback ALTER COLUMN id SET DEFAULT nextval('public.watcher_window_field_feedback_id_seq'::regclass); -- @@ -2804,6 +2926,14 @@ ALTER TABLE ONLY pgboss.warning ADD CONSTRAINT warning_pkey PRIMARY KEY (id); +-- +-- Name: _reactions_backup_2026_04_25 _reactions_backup_2026_04_25_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public._reactions_backup_2026_04_25 + ADD CONSTRAINT _reactions_backup_2026_04_25_pkey PRIMARY KEY (id, backed_up_at); + + -- -- Name: account account_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2844,6 +2974,14 @@ ALTER TABLE ONLY public.agent_grants ADD CONSTRAINT agent_grants_pkey PRIMARY KEY (id); +-- +-- Name: agent_secrets agent_secrets_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.agent_secrets + ADD CONSTRAINT agent_secrets_pkey PRIMARY KEY (name); + + -- -- Name: agent_users agent_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2909,19 +3047,19 @@ ALTER TABLE ONLY public.connector_versions -- --- Name: entities entities_organization_id_entity_type_slug_parent_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: entities entities_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.entities - ADD CONSTRAINT entities_organization_id_entity_type_slug_parent_id_key UNIQUE (organization_id, entity_type, slug, parent_id); + ADD CONSTRAINT entities_pkey PRIMARY KEY (id); -- --- Name: entities entities_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: entity_identities entity_identities_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.entities - ADD CONSTRAINT entities_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.entity_identities + ADD CONSTRAINT entity_identities_pkey PRIMARY KEY (id); -- @@ -3341,11 +3479,11 @@ ALTER TABLE ONLY public.watcher_reactions -- --- Name: watcher_window_feedback watcher_window_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback watcher_window_field_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback - ADD CONSTRAINT watcher_window_feedback_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.watcher_window_field_feedback + ADD CONSTRAINT watcher_window_field_feedback_pkey PRIMARY KEY (id); -- @@ -3476,6 +3614,20 @@ CREATE INDEX agent_connections_platform_idx ON public.agent_connections USING bt CREATE INDEX agent_grants_agent_id_idx ON public.agent_grants USING btree (agent_id); +-- +-- Name: agent_secrets_expires_at_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX agent_secrets_expires_at_idx ON public.agent_secrets USING btree (expires_at) WHERE (expires_at IS NOT NULL); + + +-- +-- Name: agent_secrets_name_prefix_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX agent_secrets_name_prefix_idx ON public.agent_secrets USING btree (name text_pattern_ops); + + -- -- Name: agents_organization_id_idx; Type: INDEX; Schema: public; Owner: - -- @@ -3728,6 +3880,13 @@ CREATE INDEX idx_entities_created_by ON public.entities USING btree (created_by) CREATE INDEX idx_entities_embedding ON public.entities USING ivfflat (embedding public.vector_cosine_ops) WITH (lists='100') WHERE ((embedding IS NOT NULL) AND (deleted_at IS NULL)); +-- +-- Name: idx_entities_entity_type_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entities_entity_type_id ON public.entities USING btree (entity_type_id) WHERE (deleted_at IS NULL); + + -- -- Name: idx_entities_metadata_domain; Type: INDEX; Schema: public; Owner: - -- @@ -3749,6 +3908,27 @@ CREATE INDEX idx_entities_name ON public.entities USING btree (lower(name)); CREATE INDEX idx_entities_organization_id ON public.entities USING btree (organization_id); +-- +-- Name: idx_entity_identities_by_entity; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_identities_by_entity ON public.entity_identities USING btree (entity_id) WHERE (deleted_at IS NULL); + + +-- +-- Name: idx_entity_identities_live_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_entity_identities_live_unique ON public.entity_identities USING btree (organization_id, namespace, identifier) WHERE (deleted_at IS NULL); + + +-- +-- Name: idx_entity_identities_lookup; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_identities_lookup ON public.entity_identities USING btree (organization_id, namespace, identifier) WHERE (deleted_at IS NULL); + + -- -- Name: idx_entity_rel_type_rules_type; Type: INDEX; Schema: public; Owner: - -- @@ -3763,6 +3943,13 @@ CREATE INDEX idx_entity_rel_type_rules_type ON public.entity_relationship_type_r CREATE UNIQUE INDEX idx_entity_rel_types_org_slug ON public.entity_relationship_types USING btree (organization_id, slug) WHERE (status = 'active'::text); +-- +-- Name: idx_entity_relationship_types_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_relationship_types_managed_by_template ON public.entity_relationship_types USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_entity_relationships_from; Type: INDEX; Schema: public; Owner: - -- @@ -3812,6 +3999,13 @@ CREATE INDEX idx_entity_type_audit_type_id ON public.entity_type_audit USING btr CREATE INDEX idx_entity_types_active ON public.entity_types USING btree (id) WHERE (deleted_at IS NULL); +-- +-- Name: idx_entity_types_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_types_managed_by_template ON public.entity_types USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_entity_types_org_slug; Type: INDEX; Schema: public; Owner: - -- @@ -3875,6 +4069,13 @@ CREATE INDEX idx_event_classifiers_entity_ids ON public.event_classifiers USING CREATE INDEX idx_event_classifiers_insight_id ON public.event_classifiers USING btree (watcher_id) WHERE (watcher_id IS NOT NULL); +-- +-- Name: idx_event_classifiers_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_event_classifiers_managed_by_template ON public.event_classifiers USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_event_classifiers_organization_id; Type: INDEX; Schema: public; Owner: - -- @@ -3987,6 +4188,55 @@ CREATE INDEX idx_events_feed_id ON public.events USING btree (feed_id); CREATE INDEX idx_events_fulltext ON public.events USING gin (to_tsvector('english'::regconfig, COALESCE(payload_text, ''::text))); +-- +-- Name: idx_events_metadata_auth_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_auth_user_id ON public.events USING btree (((metadata ->> 'auth_user_id'::text))) WHERE (metadata ? 'auth_user_id'::text); + + +-- +-- Name: idx_events_metadata_email; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_email ON public.events USING btree (((metadata ->> 'email'::text))) WHERE (metadata ? 'email'::text); + + +-- +-- Name: idx_events_metadata_github_login; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_github_login ON public.events USING btree (((metadata ->> 'github_login'::text))) WHERE (metadata ? 'github_login'::text); + + +-- +-- Name: idx_events_metadata_google_contact_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_google_contact_id ON public.events USING btree (((metadata ->> 'google_contact_id'::text))) WHERE (metadata ? 'google_contact_id'::text); + + +-- +-- Name: idx_events_metadata_phone; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_phone ON public.events USING btree (((metadata ->> 'phone'::text))) WHERE (metadata ? 'phone'::text); + + +-- +-- Name: idx_events_metadata_slack_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_slack_user_id ON public.events USING btree (((metadata ->> 'slack_user_id'::text))) WHERE (metadata ? 'slack_user_id'::text); + + +-- +-- Name: idx_events_metadata_wa_jid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_wa_jid ON public.events USING btree (((metadata ->> 'wa_jid'::text))) WHERE (metadata ? 'wa_jid'::text); + + -- -- Name: idx_events_missing_embedding_backfill; Type: INDEX; Schema: public; Owner: - -- @@ -4155,6 +4405,13 @@ CREATE INDEX idx_notifications_unread ON public.notifications USING btree (organ CREATE INDEX idx_rate_limits_updated_at ON public.rate_limits USING btree (updated_at); +-- +-- Name: idx_runs_active_auth_per_profile; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_runs_active_auth_per_profile ON public.runs USING btree (auth_profile_id) WHERE ((run_type = 'auth'::text) AND (auth_profile_id IS NOT NULL) AND (status = ANY (ARRAY['pending'::text, 'claimed'::text, 'running'::text]))); + + -- -- Name: idx_runs_active_embed_backfill_per_org; Type: INDEX; Schema: public; Owner: - -- @@ -4183,6 +4440,20 @@ CREATE UNIQUE INDEX idx_runs_active_watcher_per_watcher ON public.runs USING btr CREATE INDEX idx_runs_connection ON public.runs USING btree (connection_id); +-- +-- Name: idx_runs_created_by_user; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_runs_created_by_user ON public.runs USING btree (created_by_user_id) WHERE (created_by_user_id IS NOT NULL); + + +-- +-- Name: idx_runs_dispatched_message_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_runs_dispatched_message_id ON public.runs USING btree (dispatched_message_id) WHERE (dispatched_message_id IS NOT NULL); + + -- -- Name: idx_runs_feed; Type: INDEX; Schema: public; Owner: - -- @@ -4288,6 +4559,13 @@ CREATE INDEX idx_watcher_window_events_window ON public.watcher_window_events US CREATE INDEX idx_watcher_windows_parent ON public.watcher_windows USING btree (parent_window_id) WHERE (parent_window_id IS NOT NULL); +-- +-- Name: idx_watcher_windows_run_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_watcher_windows_run_id ON public.watcher_windows USING btree (run_id) WHERE (run_id IS NOT NULL); + + -- -- Name: idx_watcher_windows_template_version; Type: INDEX; Schema: public; Owner: - -- @@ -4337,6 +4615,13 @@ CREATE INDEX idx_watchers_created_by ON public.watchers USING btree (created_by) CREATE INDEX idx_watchers_entity_ids ON public.watchers USING gin (entity_ids); +-- +-- Name: idx_watchers_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_watchers_managed_by_template ON public.watchers USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_watchers_next_run_at; Type: INDEX; Schema: public; Owner: - -- @@ -4401,17 +4686,17 @@ CREATE INDEX idx_workers_user ON public.workers USING btree (user_id) WHERE (use -- --- Name: idx_wwf_watcher; Type: INDEX; Schema: public; Owner: - +-- Name: idx_wwff_watcher_field_recent; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX idx_wwf_watcher ON public.watcher_window_feedback USING btree (watcher_id); +CREATE INDEX idx_wwff_watcher_field_recent ON public.watcher_window_field_feedback USING btree (watcher_id, field_path, created_at DESC); -- --- Name: idx_wwf_window; Type: INDEX; Schema: public; Owner: - +-- Name: idx_wwff_window; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX idx_wwf_window ON public.watcher_window_feedback USING btree (window_id); +CREATE INDEX idx_wwff_window ON public.watcher_window_field_feedback USING btree (window_id); -- @@ -4857,6 +5142,14 @@ ALTER TABLE ONLY public.entities ADD CONSTRAINT entities_created_by_fkey FOREIGN KEY (created_by) REFERENCES public."user"(id) ON DELETE RESTRICT; +-- +-- Name: entities entities_entity_type_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entities + ADD CONSTRAINT entities_entity_type_id_fkey FOREIGN KEY (entity_type_id) REFERENCES public.entity_types(id); + + -- -- Name: entities entities_organization_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -4881,6 +5174,22 @@ ALTER TABLE ONLY public.entities ADD CONSTRAINT entities_view_template_version_fk FOREIGN KEY (current_view_template_version_id) REFERENCES public.view_template_versions(id); +-- +-- Name: entity_identities entity_identities_entity_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entity_identities + ADD CONSTRAINT entity_identities_entity_id_fkey FOREIGN KEY (entity_id) REFERENCES public.entities(id) ON DELETE CASCADE; + + +-- +-- Name: entity_identities entity_identities_organization_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entity_identities + ADD CONSTRAINT entity_identities_organization_id_fkey FOREIGN KEY (organization_id) REFERENCES public.organization(id) ON DELETE CASCADE; + + -- -- Name: entity_relationship_type_rules entity_relationship_type_rules_relationship_type_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5102,7 +5411,7 @@ ALTER TABLE ONLY public.event_embeddings -- ALTER TABLE ONLY public.events - ADD CONSTRAINT events_client_id_fkey FOREIGN KEY (client_id) REFERENCES public.oauth_clients(id); + ADD CONSTRAINT events_client_id_fkey FOREIGN KEY (client_id) REFERENCES public.oauth_clients(id) ON DELETE SET NULL; -- @@ -5409,6 +5718,14 @@ ALTER TABLE ONLY public.personal_access_tokens ADD CONSTRAINT personal_access_tokens_user_id_fkey FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE CASCADE; +-- +-- Name: runs runs_auth_profile_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.runs + ADD CONSTRAINT runs_auth_profile_id_fkey FOREIGN KEY (auth_profile_id) REFERENCES public.auth_profiles(id) ON DELETE CASCADE; + + -- -- Name: runs runs_connection_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5417,6 +5734,14 @@ ALTER TABLE ONLY public.runs ADD CONSTRAINT runs_connection_id_fkey FOREIGN KEY (connection_id) REFERENCES public.connections(id) ON DELETE SET NULL; +-- +-- Name: runs runs_created_by_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.runs + ADD CONSTRAINT runs_created_by_user_id_fkey FOREIGN KEY (created_by_user_id) REFERENCES public."user"(id) ON DELETE SET NULL; + + -- -- Name: runs runs_feed_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5506,19 +5831,27 @@ ALTER TABLE ONLY public.watcher_versions -- --- Name: watcher_window_feedback watcher_window_feedback_watcher_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback watcher_window_field_feedback_watcher_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback - ADD CONSTRAINT watcher_window_feedback_watcher_id_fkey FOREIGN KEY (watcher_id) REFERENCES public.watchers(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.watcher_window_field_feedback + ADD CONSTRAINT watcher_window_field_feedback_watcher_id_fkey FOREIGN KEY (watcher_id) REFERENCES public.watchers(id) ON DELETE CASCADE; -- --- Name: watcher_window_feedback watcher_window_feedback_window_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback watcher_window_field_feedback_window_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback - ADD CONSTRAINT watcher_window_feedback_window_id_fkey FOREIGN KEY (window_id) REFERENCES public.watcher_windows(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.watcher_window_field_feedback + ADD CONSTRAINT watcher_window_field_feedback_window_id_fkey FOREIGN KEY (window_id) REFERENCES public.watcher_windows(id) ON DELETE CASCADE; + + +-- +-- Name: watcher_windows watcher_windows_run_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.watcher_windows + ADD CONSTRAINT watcher_windows_run_id_fkey FOREIGN KEY (run_id) REFERENCES public.runs(id) ON DELETE SET NULL; -- @@ -5557,7 +5890,7 @@ ALTER TABLE ONLY public.workspace_settings -- PostgreSQL database dump complete -- -\unrestrict yPseyczHShSrIqawjVUmfWrB9Pz35TAROpf9iU3KThZVhO4FOYhotkUDqwmder9 +\unrestrict bqBE913aGPDwzJsfFpJo9ktouNunyv25fIQIyfTS9VJm4SUnYw8b0K9sqZozUWV -- @@ -5577,4 +5910,19 @@ INSERT INTO public.schema_migrations (version) VALUES ('20260402120000'), ('20260405193000'), ('20260408120000'), - ('20260408120001'); + ('20260408120001'), + ('20260409110000'), + ('20260409130000'), + ('20260410120000'), + ('20260413170000'), + ('20260416120000'), + ('20260417100000'), + ('20260418100000'), + ('20260418110000'), + ('20260419120000'), + ('20260420120000'), + ('20260424030000'), + ('20260424130000'), + ('20260425100000'), + ('20260425120000'), + ('20260426120000');