diff --git a/db/migrations/20260426120000_entities_entity_type_fk.sql b/db/migrations/20260426120000_entities_entity_type_fk.sql new file mode 100644 index 000000000..264fc56ce --- /dev/null +++ b/db/migrations/20260426120000_entities_entity_type_fk.sql @@ -0,0 +1,101 @@ +-- migrate:up + +-- Convert entities.entity_type from a text slug to an FK on entity_types(id). +-- Two motivations folded into one change: +-- +-- 1. Integrity. Today entity_types renames orphan all referencing entities +-- (slug-based reference is silent FK with no enforcement). Hard-deletes +-- bypass the validator entirely. With a real FK, Postgres refuses to +-- drop a referenced type and renames update for free (the slug becomes +-- display only — JOIN to entity_types for it). +-- +-- 2. Cross-org vocabulary. entity_types.id is globally unique (one sequence +-- across all orgs), so an entity in tenant org A can carry a type defined +-- in public-catalog org B by FK alone. No additional org_id column on +-- entities is needed once the slug-based same-org coupling is gone. +-- +-- Single-prod-DB migration: add nullable column, backfill, fail loudly on +-- orphans, set NOT NULL, drop the text column. Run manually. + +-- 1. Add the FK column, nullable for backfill. +ALTER TABLE public.entities + ADD COLUMN entity_type_id integer REFERENCES public.entity_types(id); + +-- 2. Backfill from existing (organization_id, entity_type slug) → entity_types.id. +-- Prefer live entity_types rows; fall back to soft-deleted ones to preserve +-- history. Without the ORDER BY, a slug+org pair with both an active and a +-- soft-deleted row would resolve non-deterministically — entity_types' UNIQUE +-- index on slug only covers `deleted_at IS NULL` rows, so collisions can exist. +UPDATE public.entities e +SET entity_type_id = ( + SELECT et.id + FROM public.entity_types et + WHERE et.slug = e.entity_type + AND et.organization_id = e.organization_id + ORDER BY (et.deleted_at IS NULL) DESC, et.id DESC + LIMIT 1 +) +WHERE e.entity_type_id IS NULL; + +-- 3. Fail loudly on orphans. If any entities reference a slug with no matching +-- entity_types row, that's pre-existing data corruption from the slug-based +-- regime. Surface it; don't paper over. +DO $$ +DECLARE + orphan_count integer; +BEGIN + SELECT COUNT(*) INTO orphan_count FROM public.entities WHERE entity_type_id IS NULL; + IF orphan_count > 0 THEN + RAISE EXCEPTION + 'entity_type FK migration: % entities have entity_type slugs with no matching entity_types row. Investigate before re-running.', + orphan_count; + END IF; +END $$; + +-- 4. Tighten the FK column. +ALTER TABLE public.entities + ALTER COLUMN entity_type_id SET NOT NULL; + +-- 5. Index for filter/list queries that previously used entity_type slug. +CREATE INDEX idx_entities_entity_type_id + ON public.entities (entity_type_id) + WHERE deleted_at IS NULL; + +-- 6. Drop the redundant UNIQUE constraint that referenced entity_type. The +-- stronger `entities_slug_parent_unique` (UNIQUE on org_id, COALESCE(parent_id, +-- 0), slug) already enforces slug uniqueness within (org, parent) regardless +-- of entity type, with NULL-parent collapsing — so this constraint never +-- caught anything the index didn't already catch. Drop it explicitly rather +-- than letting DROP COLUMN cascade silently. +ALTER TABLE public.entities + DROP CONSTRAINT IF EXISTS entities_organization_id_entity_type_slug_parent_id_key; + +-- 7. Drop the column comment so DROP COLUMN doesn't carry a stale doc string +-- if this migration is ever rolled back and re-applied. +COMMENT ON COLUMN public.entities.entity_type IS NULL; + +-- 8. Drop the text column. All readers JOIN to entity_types for the slug. +ALTER TABLE public.entities DROP COLUMN entity_type; + + +-- migrate:down + +ALTER TABLE public.entities ADD COLUMN entity_type text; + +UPDATE public.entities e +SET entity_type = et.slug +FROM public.entity_types et +WHERE et.id = e.entity_type_id; + +ALTER TABLE public.entities ALTER COLUMN entity_type SET NOT NULL; + +COMMENT ON COLUMN public.entities.entity_type IS + 'Type of entity: brand, product (future: location, feature, team)'; + +ALTER TABLE public.entities + ADD CONSTRAINT entities_organization_id_entity_type_slug_parent_id_key + UNIQUE (organization_id, entity_type, slug, parent_id); + +DROP INDEX IF EXISTS public.idx_entities_entity_type_id; + +ALTER TABLE public.entities DROP COLUMN entity_type_id; diff --git a/db/schema.sql b/db/schema.sql index e7bfe8f48..b7ec1b525 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,4 +1,4 @@ -\restrict yPseyczHShSrIqawjVUmfWrB9Pz35TAROpf9iU3KThZVhO4FOYhotkUDqwmder9 +\restrict bqBE913aGPDwzJsfFpJo9ktouNunyv25fIQIyfTS9VJm4SUnYw8b0K9sqZozUWV -- Dumped from database version 18.1 (Debian 18.1-1.pgdg13+2) -- Dumped by pg_dump version 18.1 (Homebrew) @@ -531,6 +531,18 @@ CREATE TABLE pgboss.warning ( ); +-- +-- Name: _reactions_backup_2026_04_25; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public._reactions_backup_2026_04_25 ( + id text NOT NULL, + reaction_script text, + reaction_script_compiled text, + backed_up_at timestamp with time zone DEFAULT now() NOT NULL +); + + -- -- Name: account; Type: TABLE; Schema: public; Owner: - -- @@ -612,6 +624,26 @@ ALTER TABLE public.agent_grants ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY ); +-- +-- Name: agent_secrets; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.agent_secrets ( + name text NOT NULL, + ciphertext text NOT NULL, + expires_at timestamp with time zone, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: TABLE agent_secrets; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.agent_secrets IS 'Encrypted secret values referenced via secret:// refs. Backs the PostgresSecretStore implementation of @lobu/gateway WritableSecretStore.'; + + -- -- Name: agent_users; Type: TABLE; Schema: public; Owner: - -- @@ -682,7 +714,8 @@ CREATE TABLE public.auth_profiles ( created_by text, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT auth_profiles_profile_kind_check CHECK ((profile_kind = ANY (ARRAY['env'::text, 'oauth_app'::text, 'oauth_account'::text, 'browser_session'::text]))), + metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + CONSTRAINT auth_profiles_profile_kind_check CHECK ((profile_kind = ANY (ARRAY['env'::text, 'oauth_app'::text, 'oauth_account'::text, 'browser_session'::text, 'interactive'::text]))), CONSTRAINT auth_profiles_status_check CHECK ((status = ANY (ARRAY['active'::text, 'pending_auth'::text, 'error'::text, 'revoked'::text]))) ); @@ -812,10 +845,19 @@ CREATE TABLE public.connector_definitions ( api_type text DEFAULT 'api'::text NOT NULL, favicon_domain text, openapi_config jsonb, + default_connection_config jsonb, + entity_link_overrides jsonb, CONSTRAINT connector_definitions_status_check CHECK ((status = ANY (ARRAY['active'::text, 'archived'::text, 'draft'::text]))) ); +-- +-- Name: COLUMN connector_definitions.entity_link_overrides; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.connector_definitions.entity_link_overrides IS 'Per-install override of connector entityLinks rules. See resolveEntityLinkRules() for merge semantics.'; + + -- -- Name: connector_definitions_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -1007,7 +1049,6 @@ CREATE VIEW public.current_event_records AS CREATE TABLE public.entities ( id bigint NOT NULL, - entity_type text NOT NULL, parent_id bigint, name text NOT NULL, metadata jsonb DEFAULT '{}'::jsonb, @@ -1022,7 +1063,8 @@ CREATE TABLE public.entities ( embedding public.vector(768), content_tsv tsvector GENERATED ALWAYS AS (to_tsvector('english'::regconfig, ((COALESCE(name, ''::text) || ' '::text) || COALESCE(content, ''::text)))) STORED, content_hash text, - deleted_at timestamp with time zone + deleted_at timestamp with time zone, + entity_type_id integer NOT NULL ); @@ -1033,13 +1075,6 @@ CREATE TABLE public.entities ( COMMENT ON TABLE public.entities IS 'Unified entity table (brands, products, and future entity types)'; --- --- Name: COLUMN entities.entity_type; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.entities.entity_type IS 'Type of entity: brand, product (future: location, feature, team)'; - - -- -- Name: COLUMN entities.parent_id; Type: COMMENT; Schema: public; Owner: - -- @@ -1073,6 +1108,70 @@ CREATE SEQUENCE public.entities_id_seq ALTER SEQUENCE public.entities_id_seq OWNED BY public.entities.id; +-- +-- Name: entity_identities; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.entity_identities ( + id bigint NOT NULL, + organization_id text NOT NULL, + entity_id bigint NOT NULL, + namespace text NOT NULL, + identifier text NOT NULL, + source_connector text, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + deleted_at timestamp with time zone +); + + +-- +-- Name: TABLE entity_identities; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.entity_identities IS 'Normalized identifier claims per entity. See docs/identity-linking.md for the full pattern.'; + + +-- +-- Name: COLUMN entity_identities.namespace; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.entity_identities.namespace IS 'Identifier kind. Standard values: phone, email, wa_jid, slack_user_id, github_login, auth_user_id, google_contact_id. Custom namespaces allowed but connectors sharing a namespace must agree on its format.'; + + +-- +-- Name: COLUMN entity_identities.identifier; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.entity_identities.identifier IS 'Normalized identifier value (E.164 digits for phone, lowercase for email, etc.). Normalizers in @lobu/owletto-sdk own the canonical form.'; + + +-- +-- Name: COLUMN entity_identities.source_connector; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.entity_identities.source_connector IS 'Who claimed this identifier: "connector:whatsapp", "manual", or null when seeded by migration.'; + + +-- +-- Name: entity_identities_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.entity_identities_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: entity_identities_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.entity_identities_id_seq OWNED BY public.entity_identities.id; + + -- -- Name: entity_relationship_type_rules; Type: TABLE; Schema: public; Owner: - -- @@ -1126,6 +1225,8 @@ CREATE TABLE public.entity_relationship_types ( deleted_at timestamp with time zone, created_at timestamp with time zone DEFAULT now(), updated_at timestamp with time zone DEFAULT now(), + managed_by_template_agent_id text, + source_template_org_id text, CONSTRAINT entity_relationship_types_status_check CHECK ((status = ANY (ARRAY['active'::text, 'archived'::text]))) ); @@ -1246,7 +1347,9 @@ CREATE TABLE public.entity_types ( created_at timestamp with time zone DEFAULT now(), updated_at timestamp with time zone DEFAULT now(), current_view_template_version_id integer, - event_kinds jsonb + event_kinds jsonb, + managed_by_template_agent_id text, + source_template_org_id text ); @@ -1379,6 +1482,8 @@ CREATE TABLE public.event_classifiers ( watcher_id bigint, organization_id text, entity_ids bigint[], + managed_by_template_agent_id text, + source_template_org_id text, CONSTRAINT event_classifiers_status_check CHECK ((status = ANY (ARRAY['active'::text, 'deprecated'::text]))) ); @@ -1870,8 +1975,12 @@ CREATE TABLE public.runs ( watcher_id integer, window_id bigint, approved_input jsonb, + auth_profile_id bigint, + auth_signal jsonb, + created_by_user_id text, + dispatched_message_id text, CONSTRAINT runs_approval_status_check CHECK ((approval_status = ANY (ARRAY['pending'::text, 'approved'::text, 'rejected'::text, 'auto'::text]))), - CONSTRAINT runs_run_type_check CHECK ((run_type = ANY (ARRAY['sync'::text, 'action'::text, 'code'::text, 'insight'::text, 'embed_backfill'::text, 'watcher'::text]))), + CONSTRAINT runs_run_type_check CHECK ((run_type = ANY (ARRAY['sync'::text, 'action'::text, 'code'::text, 'insight'::text, 'watcher'::text, 'embed_backfill'::text, 'auth'::text]))), CONSTRAINT runs_status_check CHECK ((status = ANY (ARRAY['pending'::text, 'claimed'::text, 'running'::text, 'completed'::text, 'failed'::text, 'cancelled'::text, 'timeout'::text]))) ); @@ -2268,26 +2377,29 @@ ALTER SEQUENCE public.watcher_window_content_id_seq OWNED BY public.watcher_wind -- --- Name: watcher_window_feedback; Type: TABLE; Schema: public; Owner: - +-- Name: watcher_window_field_feedback; Type: TABLE; Schema: public; Owner: - -- -CREATE TABLE public.watcher_window_feedback ( +CREATE TABLE public.watcher_window_field_feedback ( id bigint NOT NULL, window_id integer NOT NULL, watcher_id integer NOT NULL, organization_id text NOT NULL, - corrections jsonb NOT NULL, - notes text, + field_path text NOT NULL, + mutation text DEFAULT 'set'::text NOT NULL, + corrected_value jsonb, + note text, created_by text NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL + created_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT watcher_window_field_feedback_mutation_check CHECK ((mutation = ANY (ARRAY['set'::text, 'remove'::text, 'add'::text]))) ); -- --- Name: watcher_window_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- Name: watcher_window_field_feedback_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- -CREATE SEQUENCE public.watcher_window_feedback_id_seq +CREATE SEQUENCE public.watcher_window_field_feedback_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -2296,10 +2408,10 @@ CREATE SEQUENCE public.watcher_window_feedback_id_seq -- --- Name: watcher_window_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- Name: watcher_window_field_feedback_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - -- -ALTER SEQUENCE public.watcher_window_feedback_id_seq OWNED BY public.watcher_window_feedback.id; +ALTER SEQUENCE public.watcher_window_field_feedback_id_seq OWNED BY public.watcher_window_field_feedback.id; -- @@ -2323,7 +2435,8 @@ CREATE TABLE public.watcher_windows ( version_id integer, depth integer DEFAULT 0, client_id text, - run_metadata jsonb DEFAULT '{}'::jsonb NOT NULL + run_metadata jsonb DEFAULT '{}'::jsonb NOT NULL, + run_id bigint ); @@ -2400,6 +2513,8 @@ CREATE TABLE public.watchers ( scheduler_client_id text, source_watcher_id integer, watcher_group_id integer NOT NULL, + managed_by_template_agent_id text, + source_template_org_id text, CONSTRAINT insights_status_check CHECK ((status = ANY (ARRAY['active'::text, 'archived'::text]))) ); @@ -2586,6 +2701,13 @@ ALTER TABLE ONLY public.connector_versions ALTER COLUMN id SET DEFAULT nextval(' ALTER TABLE ONLY public.entities ALTER COLUMN id SET DEFAULT nextval('public.entities_id_seq'::regclass); +-- +-- Name: entity_identities id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entity_identities ALTER COLUMN id SET DEFAULT nextval('public.entity_identities_id_seq'::regclass); + + -- -- Name: entity_relationship_type_rules id; Type: DEFAULT; Schema: public; Owner: - -- @@ -2720,10 +2842,10 @@ ALTER TABLE ONLY public.watcher_window_events ALTER COLUMN id SET DEFAULT nextva -- --- Name: watcher_window_feedback id; Type: DEFAULT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback id; Type: DEFAULT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback ALTER COLUMN id SET DEFAULT nextval('public.watcher_window_feedback_id_seq'::regclass); +ALTER TABLE ONLY public.watcher_window_field_feedback ALTER COLUMN id SET DEFAULT nextval('public.watcher_window_field_feedback_id_seq'::regclass); -- @@ -2804,6 +2926,14 @@ ALTER TABLE ONLY pgboss.warning ADD CONSTRAINT warning_pkey PRIMARY KEY (id); +-- +-- Name: _reactions_backup_2026_04_25 _reactions_backup_2026_04_25_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public._reactions_backup_2026_04_25 + ADD CONSTRAINT _reactions_backup_2026_04_25_pkey PRIMARY KEY (id, backed_up_at); + + -- -- Name: account account_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2844,6 +2974,14 @@ ALTER TABLE ONLY public.agent_grants ADD CONSTRAINT agent_grants_pkey PRIMARY KEY (id); +-- +-- Name: agent_secrets agent_secrets_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.agent_secrets + ADD CONSTRAINT agent_secrets_pkey PRIMARY KEY (name); + + -- -- Name: agent_users agent_users_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2909,19 +3047,19 @@ ALTER TABLE ONLY public.connector_versions -- --- Name: entities entities_organization_id_entity_type_slug_parent_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: entities entities_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.entities - ADD CONSTRAINT entities_organization_id_entity_type_slug_parent_id_key UNIQUE (organization_id, entity_type, slug, parent_id); + ADD CONSTRAINT entities_pkey PRIMARY KEY (id); -- --- Name: entities entities_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: entity_identities entity_identities_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.entities - ADD CONSTRAINT entities_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.entity_identities + ADD CONSTRAINT entity_identities_pkey PRIMARY KEY (id); -- @@ -3341,11 +3479,11 @@ ALTER TABLE ONLY public.watcher_reactions -- --- Name: watcher_window_feedback watcher_window_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback watcher_window_field_feedback_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback - ADD CONSTRAINT watcher_window_feedback_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.watcher_window_field_feedback + ADD CONSTRAINT watcher_window_field_feedback_pkey PRIMARY KEY (id); -- @@ -3476,6 +3614,20 @@ CREATE INDEX agent_connections_platform_idx ON public.agent_connections USING bt CREATE INDEX agent_grants_agent_id_idx ON public.agent_grants USING btree (agent_id); +-- +-- Name: agent_secrets_expires_at_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX agent_secrets_expires_at_idx ON public.agent_secrets USING btree (expires_at) WHERE (expires_at IS NOT NULL); + + +-- +-- Name: agent_secrets_name_prefix_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX agent_secrets_name_prefix_idx ON public.agent_secrets USING btree (name text_pattern_ops); + + -- -- Name: agents_organization_id_idx; Type: INDEX; Schema: public; Owner: - -- @@ -3728,6 +3880,13 @@ CREATE INDEX idx_entities_created_by ON public.entities USING btree (created_by) CREATE INDEX idx_entities_embedding ON public.entities USING ivfflat (embedding public.vector_cosine_ops) WITH (lists='100') WHERE ((embedding IS NOT NULL) AND (deleted_at IS NULL)); +-- +-- Name: idx_entities_entity_type_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entities_entity_type_id ON public.entities USING btree (entity_type_id) WHERE (deleted_at IS NULL); + + -- -- Name: idx_entities_metadata_domain; Type: INDEX; Schema: public; Owner: - -- @@ -3749,6 +3908,27 @@ CREATE INDEX idx_entities_name ON public.entities USING btree (lower(name)); CREATE INDEX idx_entities_organization_id ON public.entities USING btree (organization_id); +-- +-- Name: idx_entity_identities_by_entity; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_identities_by_entity ON public.entity_identities USING btree (entity_id) WHERE (deleted_at IS NULL); + + +-- +-- Name: idx_entity_identities_live_unique; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_entity_identities_live_unique ON public.entity_identities USING btree (organization_id, namespace, identifier) WHERE (deleted_at IS NULL); + + +-- +-- Name: idx_entity_identities_lookup; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_identities_lookup ON public.entity_identities USING btree (organization_id, namespace, identifier) WHERE (deleted_at IS NULL); + + -- -- Name: idx_entity_rel_type_rules_type; Type: INDEX; Schema: public; Owner: - -- @@ -3763,6 +3943,13 @@ CREATE INDEX idx_entity_rel_type_rules_type ON public.entity_relationship_type_r CREATE UNIQUE INDEX idx_entity_rel_types_org_slug ON public.entity_relationship_types USING btree (organization_id, slug) WHERE (status = 'active'::text); +-- +-- Name: idx_entity_relationship_types_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_relationship_types_managed_by_template ON public.entity_relationship_types USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_entity_relationships_from; Type: INDEX; Schema: public; Owner: - -- @@ -3812,6 +3999,13 @@ CREATE INDEX idx_entity_type_audit_type_id ON public.entity_type_audit USING btr CREATE INDEX idx_entity_types_active ON public.entity_types USING btree (id) WHERE (deleted_at IS NULL); +-- +-- Name: idx_entity_types_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_entity_types_managed_by_template ON public.entity_types USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_entity_types_org_slug; Type: INDEX; Schema: public; Owner: - -- @@ -3875,6 +4069,13 @@ CREATE INDEX idx_event_classifiers_entity_ids ON public.event_classifiers USING CREATE INDEX idx_event_classifiers_insight_id ON public.event_classifiers USING btree (watcher_id) WHERE (watcher_id IS NOT NULL); +-- +-- Name: idx_event_classifiers_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_event_classifiers_managed_by_template ON public.event_classifiers USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_event_classifiers_organization_id; Type: INDEX; Schema: public; Owner: - -- @@ -3987,6 +4188,55 @@ CREATE INDEX idx_events_feed_id ON public.events USING btree (feed_id); CREATE INDEX idx_events_fulltext ON public.events USING gin (to_tsvector('english'::regconfig, COALESCE(payload_text, ''::text))); +-- +-- Name: idx_events_metadata_auth_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_auth_user_id ON public.events USING btree (((metadata ->> 'auth_user_id'::text))) WHERE (metadata ? 'auth_user_id'::text); + + +-- +-- Name: idx_events_metadata_email; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_email ON public.events USING btree (((metadata ->> 'email'::text))) WHERE (metadata ? 'email'::text); + + +-- +-- Name: idx_events_metadata_github_login; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_github_login ON public.events USING btree (((metadata ->> 'github_login'::text))) WHERE (metadata ? 'github_login'::text); + + +-- +-- Name: idx_events_metadata_google_contact_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_google_contact_id ON public.events USING btree (((metadata ->> 'google_contact_id'::text))) WHERE (metadata ? 'google_contact_id'::text); + + +-- +-- Name: idx_events_metadata_phone; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_phone ON public.events USING btree (((metadata ->> 'phone'::text))) WHERE (metadata ? 'phone'::text); + + +-- +-- Name: idx_events_metadata_slack_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_slack_user_id ON public.events USING btree (((metadata ->> 'slack_user_id'::text))) WHERE (metadata ? 'slack_user_id'::text); + + +-- +-- Name: idx_events_metadata_wa_jid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_metadata_wa_jid ON public.events USING btree (((metadata ->> 'wa_jid'::text))) WHERE (metadata ? 'wa_jid'::text); + + -- -- Name: idx_events_missing_embedding_backfill; Type: INDEX; Schema: public; Owner: - -- @@ -4155,6 +4405,13 @@ CREATE INDEX idx_notifications_unread ON public.notifications USING btree (organ CREATE INDEX idx_rate_limits_updated_at ON public.rate_limits USING btree (updated_at); +-- +-- Name: idx_runs_active_auth_per_profile; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_runs_active_auth_per_profile ON public.runs USING btree (auth_profile_id) WHERE ((run_type = 'auth'::text) AND (auth_profile_id IS NOT NULL) AND (status = ANY (ARRAY['pending'::text, 'claimed'::text, 'running'::text]))); + + -- -- Name: idx_runs_active_embed_backfill_per_org; Type: INDEX; Schema: public; Owner: - -- @@ -4183,6 +4440,20 @@ CREATE UNIQUE INDEX idx_runs_active_watcher_per_watcher ON public.runs USING btr CREATE INDEX idx_runs_connection ON public.runs USING btree (connection_id); +-- +-- Name: idx_runs_created_by_user; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_runs_created_by_user ON public.runs USING btree (created_by_user_id) WHERE (created_by_user_id IS NOT NULL); + + +-- +-- Name: idx_runs_dispatched_message_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_runs_dispatched_message_id ON public.runs USING btree (dispatched_message_id) WHERE (dispatched_message_id IS NOT NULL); + + -- -- Name: idx_runs_feed; Type: INDEX; Schema: public; Owner: - -- @@ -4288,6 +4559,13 @@ CREATE INDEX idx_watcher_window_events_window ON public.watcher_window_events US CREATE INDEX idx_watcher_windows_parent ON public.watcher_windows USING btree (parent_window_id) WHERE (parent_window_id IS NOT NULL); +-- +-- Name: idx_watcher_windows_run_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_watcher_windows_run_id ON public.watcher_windows USING btree (run_id) WHERE (run_id IS NOT NULL); + + -- -- Name: idx_watcher_windows_template_version; Type: INDEX; Schema: public; Owner: - -- @@ -4337,6 +4615,13 @@ CREATE INDEX idx_watchers_created_by ON public.watchers USING btree (created_by) CREATE INDEX idx_watchers_entity_ids ON public.watchers USING gin (entity_ids); +-- +-- Name: idx_watchers_managed_by_template; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_watchers_managed_by_template ON public.watchers USING btree (managed_by_template_agent_id) WHERE (managed_by_template_agent_id IS NOT NULL); + + -- -- Name: idx_watchers_next_run_at; Type: INDEX; Schema: public; Owner: - -- @@ -4401,17 +4686,17 @@ CREATE INDEX idx_workers_user ON public.workers USING btree (user_id) WHERE (use -- --- Name: idx_wwf_watcher; Type: INDEX; Schema: public; Owner: - +-- Name: idx_wwff_watcher_field_recent; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX idx_wwf_watcher ON public.watcher_window_feedback USING btree (watcher_id); +CREATE INDEX idx_wwff_watcher_field_recent ON public.watcher_window_field_feedback USING btree (watcher_id, field_path, created_at DESC); -- --- Name: idx_wwf_window; Type: INDEX; Schema: public; Owner: - +-- Name: idx_wwff_window; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX idx_wwf_window ON public.watcher_window_feedback USING btree (window_id); +CREATE INDEX idx_wwff_window ON public.watcher_window_field_feedback USING btree (window_id); -- @@ -4857,6 +5142,14 @@ ALTER TABLE ONLY public.entities ADD CONSTRAINT entities_created_by_fkey FOREIGN KEY (created_by) REFERENCES public."user"(id) ON DELETE RESTRICT; +-- +-- Name: entities entities_entity_type_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entities + ADD CONSTRAINT entities_entity_type_id_fkey FOREIGN KEY (entity_type_id) REFERENCES public.entity_types(id); + + -- -- Name: entities entities_organization_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -4881,6 +5174,22 @@ ALTER TABLE ONLY public.entities ADD CONSTRAINT entities_view_template_version_fk FOREIGN KEY (current_view_template_version_id) REFERENCES public.view_template_versions(id); +-- +-- Name: entity_identities entity_identities_entity_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entity_identities + ADD CONSTRAINT entity_identities_entity_id_fkey FOREIGN KEY (entity_id) REFERENCES public.entities(id) ON DELETE CASCADE; + + +-- +-- Name: entity_identities entity_identities_organization_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entity_identities + ADD CONSTRAINT entity_identities_organization_id_fkey FOREIGN KEY (organization_id) REFERENCES public.organization(id) ON DELETE CASCADE; + + -- -- Name: entity_relationship_type_rules entity_relationship_type_rules_relationship_type_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5102,7 +5411,7 @@ ALTER TABLE ONLY public.event_embeddings -- ALTER TABLE ONLY public.events - ADD CONSTRAINT events_client_id_fkey FOREIGN KEY (client_id) REFERENCES public.oauth_clients(id); + ADD CONSTRAINT events_client_id_fkey FOREIGN KEY (client_id) REFERENCES public.oauth_clients(id) ON DELETE SET NULL; -- @@ -5409,6 +5718,14 @@ ALTER TABLE ONLY public.personal_access_tokens ADD CONSTRAINT personal_access_tokens_user_id_fkey FOREIGN KEY (user_id) REFERENCES public."user"(id) ON DELETE CASCADE; +-- +-- Name: runs runs_auth_profile_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.runs + ADD CONSTRAINT runs_auth_profile_id_fkey FOREIGN KEY (auth_profile_id) REFERENCES public.auth_profiles(id) ON DELETE CASCADE; + + -- -- Name: runs runs_connection_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5417,6 +5734,14 @@ ALTER TABLE ONLY public.runs ADD CONSTRAINT runs_connection_id_fkey FOREIGN KEY (connection_id) REFERENCES public.connections(id) ON DELETE SET NULL; +-- +-- Name: runs runs_created_by_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.runs + ADD CONSTRAINT runs_created_by_user_id_fkey FOREIGN KEY (created_by_user_id) REFERENCES public."user"(id) ON DELETE SET NULL; + + -- -- Name: runs runs_feed_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -5506,19 +5831,27 @@ ALTER TABLE ONLY public.watcher_versions -- --- Name: watcher_window_feedback watcher_window_feedback_watcher_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback watcher_window_field_feedback_watcher_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback - ADD CONSTRAINT watcher_window_feedback_watcher_id_fkey FOREIGN KEY (watcher_id) REFERENCES public.watchers(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.watcher_window_field_feedback + ADD CONSTRAINT watcher_window_field_feedback_watcher_id_fkey FOREIGN KEY (watcher_id) REFERENCES public.watchers(id) ON DELETE CASCADE; -- --- Name: watcher_window_feedback watcher_window_feedback_window_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: watcher_window_field_feedback watcher_window_field_feedback_window_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.watcher_window_feedback - ADD CONSTRAINT watcher_window_feedback_window_id_fkey FOREIGN KEY (window_id) REFERENCES public.watcher_windows(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.watcher_window_field_feedback + ADD CONSTRAINT watcher_window_field_feedback_window_id_fkey FOREIGN KEY (window_id) REFERENCES public.watcher_windows(id) ON DELETE CASCADE; + + +-- +-- Name: watcher_windows watcher_windows_run_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.watcher_windows + ADD CONSTRAINT watcher_windows_run_id_fkey FOREIGN KEY (run_id) REFERENCES public.runs(id) ON DELETE SET NULL; -- @@ -5557,7 +5890,7 @@ ALTER TABLE ONLY public.workspace_settings -- PostgreSQL database dump complete -- -\unrestrict yPseyczHShSrIqawjVUmfWrB9Pz35TAROpf9iU3KThZVhO4FOYhotkUDqwmder9 +\unrestrict bqBE913aGPDwzJsfFpJo9ktouNunyv25fIQIyfTS9VJm4SUnYw8b0K9sqZozUWV -- @@ -5577,4 +5910,19 @@ INSERT INTO public.schema_migrations (version) VALUES ('20260402120000'), ('20260405193000'), ('20260408120000'), - ('20260408120001'); + ('20260408120001'), + ('20260409110000'), + ('20260409130000'), + ('20260410120000'), + ('20260413170000'), + ('20260416120000'), + ('20260417100000'), + ('20260418100000'), + ('20260418110000'), + ('20260419120000'), + ('20260420120000'), + ('20260424030000'), + ('20260424130000'), + ('20260425100000'), + ('20260425120000'), + ('20260426120000'); diff --git a/docs/plans/world-model.md b/docs/plans/world-model.md new file mode 100644 index 000000000..58661c589 --- /dev/null +++ b/docs/plans/world-model.md @@ -0,0 +1,331 @@ +# World Model + +Long-term shape of how knowledge, identity, and templates are organized across +tenants and public catalogs in Lobu. + +## TL;DR + +- Two org kinds (`tenant`, `public_catalog`), two visibilities (`private`, `public`). +- One graph: `entities` + `entity_relationships` + `entity_identities` are the + universal primitives. Orgs are trust slices through that graph. +- Cross-org relationships allowed in **one direction only**: tenant → public_catalog. +- Templates are entities of type `agent_template` in a public_catalog org. No + schema cloning on install — agents read vocabularies from public catalogs at + runtime. +- Contribution to public knowledge happens by inviting the public org's admin + agent into your private org as a member. Existing membership/messaging + primitives, no draft tables, no contributor roles. +- No Postgres RLS in phase 1. App-level org-scoped queries (already in place) + plus a write-side guard on relationship inserts are sufficient given the + one-directional reference rule. + +## Cleanup before phase 1 + +| PR | Action | Why | +| --- | --- | --- | +| #351 — `managed_by_template_agent_id` + `source_template_org_id` columns | **Close** | No mirroring → no tracking columns | +| #353 — `installAgentFromTemplate` schema mirror (1221 LOC) | **Close** | Schema lives in public catalogs; agents reference, don't clone | +| #357 — POST /api/install | Trim to ~30 lines | Just inserts an agent row in tenant + provisions identity | +| #359 — identity provisioning ($member + wa_jid) | Keep | Orthogonal — identities are real regardless | +| #362 — install manifest | Trim | Drop the env-var slug→bot-phone map; bot phone moves to data on the template entity | + +Also revisit #358 (company-aware world model for personal-finance) against this +plan once it lands — its direction is compatible but its details may need to be +re-aligned. + +## Long-term primitives + +| Primitive | Purpose | Stable? | +| --- | --- | --- | +| `organization` (with `kind` + `visibility`) | Trust boundary | Yes | +| `entities` (typed rows, scoped to one org) | Anything: $member, a company, a tax filing, an agent template, a review | Yes | +| `entity_types` + `entity_relationship_types` | Vocabulary, **data not schema** — templates ship new types as rows, not migrations | Yes | +| `entity_relationships` (typed edges, explicit `source_organization_id` + `target_organization_id`) | All semantic facts, references, forks, reviews | Yes | +| `entity_identities` (namespace + identifier → entity) | Technical lookup keys (auth_user_id, email, wa_jid, uk_utr, uk_ni, companies-house-number) | Yes | + +UUIDs everywhere — federation across instances or third-party catalogs becomes +cheap to add later. + +## Org topology + +- **`tenant`** — user's private space. `visibility=private`. Personal data, + installed agents, filings, message history. +- **`public_catalog`** — curated public knowledge & published artifacts. + `visibility=public`. Companies, gov bodies, currencies, tax years, allowance + definitions, agent templates, skill definitions, reviews. + +Three kinds of orgs collapsed to two: there is no `template` org kind. Templates +are entities of type `agent_template` in some public_catalog org, distinguished +by entity type, not org kind. + +## Cross-org references + +- Direction: tenant → public_catalog only (one-way). +- Read paths never mix scopes: queries either hit the user's org + (membership-scoped) or public orgs (`visibility=public` filter), never both at + once. This is what removes the "every read site must remember `OR + visibility=public`" risk. +- Write-side guard at the application layer: when inserting an + `entity_relationship`, validate that `target_organization_id` is either the + same org as the source OR an org with `visibility='public'`. A Postgres + trigger version of the same check is cheap defense-in-depth if/when needed. +- RLS is **not required** for this model to be safe. It remains a sensible + defense-in-depth project for later but is decoupled from world-model + delivery. + +## Templates + +A template is an entity of type `agent_template`: + +- Carries: system prompt, model config, tool list, skill manifest, version, + bot phone, descriptive metadata. +- References public catalogs it operates over via relationships: + `uses_catalog` → `public-uk-tax`, `uses_catalog` → `public-uk-finance`. +- Authorship, forks, reviews, ratings: `entity_relationships` (`authored_by`, + `forked_from`, `reviews`, `rated`). + +Installation: + +1. Insert agent row in user's tenant org with `template_entity_id`. +2. Provision `$member` if missing (identity provisioning logic from #359). +3. Done. No schema cloning. + +When the agent boots, it builds a **schema search path**: + +- The user's tenant org (for any custom types the user added). +- The public catalogs declared by the template's `uses_catalog` relationships. + +Vocabulary updates propagate automatically — when `public-uk-tax` adds a new +type, all agents reading it pick up the new vocabulary on next boot. Catalog +versioning is explicit at the type level (e.g. `tax_filing@2024-25` and +`tax_filing@2025-26` are separate `entity_types` rows). + +## Identity + +- One `$member` entity per (org, user). Lazy-created on first meaningful + interaction in that org, not browse. +- `entity_identities` holds technical IDs against `$member`: `auth_user_id`, + `email`, `wa_jid`, `phone`, `uk_utr`, `uk_ni`, etc. Each is + (namespace, identifier) → entity_id. +- Service agents (e.g. a public org's admin agent) get their own identities + with a `service_agent` namespace so they can be invited into private orgs + the same way human users are. + +`entity_identities` (technical lookup) and `entity_relationships` (semantic +facts) stay as separate tables. Don't conflate. + +## Contribution flow + +1. User has data in their private org they think the public catalog should + know about. +2. User notifies the public org's admin agent via the existing chat/messaging + path: *"FYI I've recorded ``."* +3. If the admin agent decides it's worth canonicalizing, it requests read + access. +4. User invites the admin agent into their private org as a `viewer` + (read-only) or `collaborator` (limited write — for the agent to stamp a + "synced as ``" reference back on the user's entity). +5. Admin's agent reads, decides, writes the canonical entity into the public + org. +6. User revokes membership when done. Or leaves the agent in for ongoing sync. + +What this gives for free: + +| Need | Reuses | +| --- | --- | +| Access mechanism | Existing org membership + role | +| Audit trail | Membership invite/revoke events | +| Revocation | DELETE membership | +| Granularity | Org boundary itself (or split a sharing sub-org for narrower control) | +| Trust UX | "Invite person/agent to org" — already familiar from Slack/Drive | + +Trade-off: invitation grants whole-org read access, not per-entity. Acceptable — +coarse and explicit beats fine-grained and hidden for trust. Users wanting +narrower control can keep contribution-bound entities in a dedicated sharing +sub-org. + +## Use case: tax return + +**`public-uk-tax`** (public_catalog) seeds: + +- HMRC, the £, tax years (2024-25, 2025-26 …), tax forms (SA100, SA102, SA105, + SA108), allowance & relief definitions, filing deadlines linked to years. + +**`public-uk-finance`** (public_catalog) seeds: + +- Major banks, large PAYE-using employers (when known), the FCA, Companies + House. + +**User's tenant org** holds: + +- `$member` with identities `auth_user_id`, `email`, `uk_utr`, `uk_ni`. +- One `tax_filing` entity per year. Relationships: `for_tax_year` → public tax + year, `filed_with` → HMRC, `taxpayer` → `$member`, `includes_form` → + form-instance entities. +- `income` entities (salary, dividends, interest), each with `source` → + bank/employer in public-uk-finance. +- `expense` entities, `allowance_claim` entities pointing at public allowance + definitions. + +The agent's job at filing time is a graph walk: from `$member` → `taxpayer` → +filing → income/expense relationships, resolving sources via cross-org +references. + +## Use case: agent community + +**`public-templates`** (public_catalog) holds one entity per published template +(type `agent_template`): + +- Forks: `forked_from` between template entities. +- Versions: either entity-per-version with `next_version` edges, or + `template_version` child entities. Both fit the graph. +- Authorship: `authored_by` from template → `$member` of the author in some + org. + +**`public-community`** (public_catalog, separate org for policy reasons): + +- `review` entities, with `reviews` → template, `authored_by` → `$member`. +- Ratings as entity properties or `rated` relationships with numeric metadata. +- Tags / categories as entities with `tagged` relationships. + +Splitting `public-templates` and `public-community` reflects different admin +policies (templates are author-editable; reviews are write-once-by-author) +without inventing new permission machinery. + +## Phase 1 — implementation scope + +1. Migration: add `kind` + `visibility` columns to `organization`. Add explicit + `source_organization_id` + `target_organization_id` columns to + `entity_relationships`. +2. App-level write guard on `entity_relationships` inserts: target must be + same-org or `visibility='public'`. +3. Seed `public-uk-tax` and `public-uk-finance` orgs with canonical UK + entities. +4. Slim install endpoint (replaces #357's mirroring): insert agent row in + tenant + provision `$member` identity. +5. Identity provisioning (keep #359). +6. Search endpoint scoped to `visibility=public` orgs. +7. Update template authoring: templates become entities of type + `agent_template` in a public catalog with `uses_catalog` relationships. + Bot phone moves from env to `agent_template` metadata. + +## Deferred (with rationale) + +| Deferred | Why now isn't the right time | Cheap to add later? | +| --- | --- | --- | +| Postgres RLS | Not required given one-directional refs + scope-local reads. App-level enforcement already in place. | Yes — separate project | +| Claims (verification status machine, evidence refs, expiry, dispute primitives, permissions table) | Not needed for tax return or initial community. Real complexity (per pi: status machine, cardinality per type, dispute states, permission projections). | Yes — additive columns + new permission table | +| Aliases / merges / tombstones for canonical entities | Needed at meaningful catalog scale. Premature now. | Yes — new tables, no existing-row migration | +| Federation (cross-instance entity references) | No multi-instance need yet. UUIDs from day one keep this option open. | Yes | +| Fine-grained per-entity sharing | Whole-org invite is coarser but explicit; serves the immediate need. | Yes — sub-orgs are the escape hatch | + +## Long-term invariants worth preserving + +1. **Vocabulary-as-data** — adding entity types or relationship types is an + INSERT, not a migration. +2. **UUIDs everywhere** — keeps federation cheap. +3. **One graph, many orgs** — orgs are trust slices through one universal + graph. +4. **Cross-org references unidirectional** (tenant → public). +5. **`entity_identities` (technical) ≠ `entity_relationships` (semantic)** — + keep them separate. + +## Implementation arc — finishing the existing work + +Status of in-flight PRs and how each lands under the new model. + +### Wave 1 — independent, ready to land now + +These don't depend on the world-model schema and aren't affected by the +template-cloning rollback. Land in any order. + +| PR | Title | Notes | +| --- | --- | --- | +| #352 | personal-org-on-signup | Creates `tenant`-kind org for new users. As-is. | +| #350 | personal-finance example | Pure content under `/agents/personal-finance/`. As-is. | +| #354 | SA100 assembly playbook | Content. As-is. | +| #355 | statement ingestion playbook | Content. As-is. | +| #356 | personal-finance evals | Content. As-is. | +| #348 | multi-org execute MCP tools | Orthogonal scaffolding. As-is. | + +### Wave 2 — world-model schema (new branches) + +Two small PRs, sequential. Total ~200 LOC including migrations + tests. + +| Branch | Scope | +| --- | --- | +| `feat/world-model-orgs` | Add `organization.kind` (`tenant | public_catalog`) + `organization.visibility` (`private | public`). Default existing rows to `tenant`/`private`. | +| `feat/world-model-relations` | Add `entity_relationships.source_organization_id` + `target_organization_id`. Backfill from current implicit scoping. App-level write guard helper rejecting cross-org targets unless target org is `public_catalog`. | + +### Wave 3 — public catalog seeding (re-targeted existing work) + +The vocabulary already in flight maps cleanly onto public catalogs. + +| PR | Re-targeting | +| --- | --- | +| #358 — company-aware world model | Re-target so the personal-finance template **org** becomes a `public_catalog` org (`kind=public_catalog`, `visibility=public`). YAML content lands as `entity_types` rows in that org. No content rewrite needed; only the seed pipeline & org metadata change. | +| #360 — phase 2 schema (FX, allowance windows, filing timeline) | Same treatment. Stacks on #358. | +| `feat/agent-template-entity` (new) | Define the `agent_template` entity type. Seed the personal-finance `agent_template` entity with metadata (system prompt, model, skill list, **bot phone**) and `uses_catalog` relationships pointing at the catalog orgs from #358/#360. | + +### Wave 4 — slim install + identity (replacing #357 / #362, keeping #359's helper) + +Three PRs, all stacked on Wave 2 + Wave 3. + +| Branch | Scope | Replaces | +| --- | --- | --- | +| `feat/slim-install` | `POST /api/install` accepts `{ template_entity_id, whatsapp_phone? }` (or slug → server-resolved to template entity). Inserts agent row in user's tenant with `template_entity_id`. Returns redirect. ~50 LOC. | #357 | +| `feat/identity-provisioning` | Salvage from #359: keep `auth/subject-identities.ts` (the helpers + signup-hook call). Drop the install-routes changes — slim-install owns those. Rebase onto `feat/slim-install`. | trims #359 | +| `feat/install-manifest-data` | `GET /api/install/manifest/:slug` reads from the `agent_template` entity. No env vars. | #362 | + +### Wave 5 — landing page + +Two-PR ship (per AGENTS.md submodule rule): + +1. owletto-web PR (the existing #20 there): update POST body to send + `template_entity_id`. Land first. +2. Parent submodule-bump PR. + +### Cleanup as PRs land + +- Close #357 when `feat/slim-install` lands. +- Close #362 when `feat/install-manifest-data` lands. +- Reduce #359 to just the salvageable helper or close + land via + `feat/identity-provisioning`. +- Delete local worktrees for `feat/install-endpoint` and + `feat/schema-mirror-install-flow` once detached. + +### Parallelism / sequencing summary + +``` +Wave 1 (any order) ─────── land independently + │ +Wave 2 ─── orgs ──── relations + │ +Wave 3 ─── #358 ── #360 ── agent-template-entity + │ +Wave 4 ─── slim-install ── identity-provisioning + │ + └── install-manifest-data + │ +Wave 5 ─── owletto-web#20 ── parent bump +``` + +Wave 1 lands now. Waves 2 and 3 can be developed in parallel by different +agents (different files), as long as 3 lands after 2's columns exist. Wave 4 +stacks on both. + +### Open questions to resolve before Wave 3 + +1. **Template org** — does personal-finance live in its own catalog org, in + `public-uk-tax`, or in a `public-templates` org? Recommendation: own org + (`public-personal-finance` or `public-templates`) for clean admin policy + separation. The `agent_template` entity references the *vocabulary* catalogs + via `uses_catalog`, so co-locating with the vocabulary isn't required. +2. **Seed mechanism** — the existing `/agents/personal-finance/` YAML + pipeline needs a small adapter so YAMLs become entity_types/entity rows in + the public catalog org. Currently they get cloned per tenant via + `installAgentFromTemplate` (now closed). Adapter is straightforward but + needs to land before #358's content can flow through. +3. **Slug → template_entity_id resolution** — keep slugs in the URL + (`/install/personal-finance`) but resolve server-side. Slug becomes a + property on the `agent_template` entity. No env-var map. diff --git a/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts b/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts index bacc7f6a8..ed2b41b11 100644 --- a/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts +++ b/packages/owletto-backend/src/__tests__/integration/connectors/whatsapp-entity-links.test.ts @@ -80,8 +80,9 @@ describe('whatsapp connector > entityLinks', () => { const entitiesAfterFirst = await sql< { id: number; name: string; metadata: Record }[] >` - SELECT id, name, metadata FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT e.id, e.name, e.metadata FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entitiesAfterFirst).toHaveLength(1); expect(entitiesAfterFirst[0].name).toBe('Alex'); @@ -120,8 +121,9 @@ describe('whatsapp connector > entityLinks', () => { }); const countAfterSecond = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(countAfterSecond[0].count).toBe('1'); }); @@ -149,8 +151,9 @@ describe('whatsapp connector > entityLinks', () => { }); const count = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(count[0].count).toBe('0'); }); @@ -185,8 +188,9 @@ describe('whatsapp connector > entityLinks', () => { }); const count = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(count[0].count).toBe('0'); }); diff --git a/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts b/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts index 1d548dea1..2b5842cf6 100644 --- a/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts +++ b/packages/owletto-backend/src/__tests__/integration/entities/member-email-redaction.test.ts @@ -48,11 +48,11 @@ describe('$member visibility policy on public orgs', () => { await sql` INSERT INTO entities ( - name, slug, entity_type, organization_id, metadata, created_by, created_at, updated_at + name, slug, entity_type_id, organization_id, metadata, created_by, created_at, updated_at ) VALUES ( 'Plain Member', 'plain-member', - '$member', + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${publicOrg.id} AND deleted_at IS NULL), ${publicOrg.id}, ${sql.json({ email: MEMBER_EMAIL, status: 'active', role: 'member' })}, ${adminUser.id}, diff --git a/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts b/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts index 6286a39f7..4a5d577d7 100644 --- a/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts +++ b/packages/owletto-backend/src/__tests__/setup/test-fixtures.ts @@ -277,11 +277,31 @@ export async function createTestEntity(options: { } } + // Tests routinely create entities in fresh orgs without first calling + // seedSystemEntityTypes(); ensure the requested type exists so the FK + // (entities.entity_type_id) resolves without forcing every test to seed. + const entityTypeSlug = options.entity_type || 'brand'; + let typeRows = await sql<{ id: number }[]>` + SELECT id FROM entity_types + WHERE slug = ${entityTypeSlug} + AND organization_id = ${options.organization_id} + AND deleted_at IS NULL + LIMIT 1 + `; + if (typeRows.length === 0) { + typeRows = await sql<{ id: number }[]>` + INSERT INTO entity_types (organization_id, slug, name, created_at, updated_at) + VALUES (${options.organization_id}, ${entityTypeSlug}, ${entityTypeSlug}, current_timestamp, current_timestamp) + RETURNING id + `; + } + const entityTypeId = typeRows[0].id; + const [inserted] = await sql` INSERT INTO entities ( name, slug, - entity_type, + entity_type_id, organization_id, parent_id, metadata, @@ -291,7 +311,7 @@ export async function createTestEntity(options: { ) VALUES ( ${options.name}, ${slug}, - ${options.entity_type || 'brand'}, + ${entityTypeId}, ${options.organization_id}, ${options.parent_id || null}, ${sql.json(metadata)}, diff --git a/packages/owletto-backend/src/auth/subject-identities.ts b/packages/owletto-backend/src/auth/subject-identities.ts index 8b2942433..0deaa2bd1 100644 --- a/packages/owletto-backend/src/auth/subject-identities.ts +++ b/packages/owletto-backend/src/auth/subject-identities.ts @@ -55,11 +55,13 @@ async function findMemberEntityIdByEmail( ): Promise { const { emailField } = await resolveMemberSchemaFields(organizationId); const rows = await sql.unsafe( - `SELECT id FROM entities - WHERE entity_type = '$member' - AND organization_id = $1 - AND metadata->>$2 = $3 - AND deleted_at IS NULL + `SELECT e.id + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = '$member' + AND e.organization_id = $1 + AND e.metadata->>$2 = $3 + AND e.deleted_at IS NULL LIMIT 1`, [organizationId, emailField, email] ); diff --git a/packages/owletto-backend/src/index.ts b/packages/owletto-backend/src/index.ts index f4a6416eb..fbe0b3213 100644 --- a/packages/owletto-backend/src/index.ts +++ b/packages/owletto-backend/src/index.ts @@ -698,18 +698,19 @@ app.get('/api/:orgSlug/watchers/windows/:windowId', mcpAuth, async (c) => { i.slug as watcher_slug, i.name as watcher_name, e.name as entity_name, - e.entity_type, + et.slug AS entity_type, parent.name as parent_name, CAST(COUNT(iwf.event_id) AS INTEGER) as content_count FROM watcher_windows iw JOIN watchers i ON iw.watcher_id = i.id JOIN entities e ON e.id = ANY(i.entity_ids) + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id LEFT JOIN watcher_window_events iwf ON iwf.window_id = iw.id WHERE iw.id = ${windowId} AND e.organization_id = ${organizationId} AND i.status = 'active' - GROUP BY iw.id, i.entity_ids, i.slug, i.name, e.name, e.entity_type, parent.name + GROUP BY iw.id, i.entity_ids, i.slug, i.name, e.name, et.slug, parent.name `; if (windowResult.length === 0) { diff --git a/packages/owletto-backend/src/public-pages.ts b/packages/owletto-backend/src/public-pages.ts index 51719a007..cf8eb96a3 100644 --- a/packages/owletto-backend/src/public-pages.ts +++ b/packages/owletto-backend/src/public-pages.ts @@ -352,7 +352,7 @@ async function getPublicEntityType( SELECT COUNT(*)::int FROM entities e WHERE e.organization_id = et.organization_id - AND e.entity_type = et.slug + AND e.entity_type_id = et.id AND e.deleted_at IS NULL ) AS entity_count FROM entity_types et @@ -1026,12 +1026,13 @@ export async function buildSitemapEntries(origin: string): Promise> { const sql = getDb(); const rows = await sql` - SELECT entity_type, COUNT(*)::int as entity_count - FROM entities - WHERE organization_id = ${organizationId} - AND deleted_at IS NULL - GROUP BY entity_type + SELECT et.slug AS entity_type, COUNT(*)::int as entity_count + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + GROUP BY et.slug `; const counts = new Map(); for (const row of rows) { @@ -301,10 +302,11 @@ async function getEntityCountForType(slug: string, organizationId: string): Prom const sql = getDb(); const rows = await sql` SELECT COUNT(*)::int as count - FROM entities - WHERE entity_type = ${slug} - AND organization_id = ${organizationId} - AND deleted_at IS NULL + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = ${slug} + AND e.organization_id = ${organizationId} + AND e.deleted_at IS NULL `; return Number(rows[0]?.count || 0); } diff --git a/packages/owletto-backend/src/tools/admin/manage_watchers.ts b/packages/owletto-backend/src/tools/admin/manage_watchers.ts index a08e36aae..71565c4b7 100644 --- a/packages/owletto-backend/src/tools/admin/manage_watchers.ts +++ b/packages/owletto-backend/src/tools/admin/manage_watchers.ts @@ -902,10 +902,12 @@ async function handleCreate( const entityResult = await sql` SELECT - e.id, e.entity_type, e.parent_id, e.slug, e.organization_id, - parent.slug as parent_slug, parent.entity_type as parent_entity_type + e.id, et.slug AS entity_type, e.parent_id, e.slug, e.organization_id, + parent.slug as parent_slug, pet.slug as parent_entity_type FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE e.id = ${entityId} `; if (entityResult.length === 0) { @@ -1059,7 +1061,10 @@ async function handleCreateFromVersion( // Fetch entity names for name pattern substitution const entityRows = await sql` - SELECT id, name, entity_type, slug FROM entities WHERE id = ANY(${`{${args.entity_ids.join(',')}}`}::bigint[]) + SELECT e.id, e.name, et.slug AS entity_type, e.slug + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ANY(${`{${args.entity_ids.join(',')}}`}::bigint[]) `; const entityMap = new Map(entityRows.map((e: any) => [Number(e.id), e])); @@ -1791,7 +1796,12 @@ async function handleCompleteWindow( const eIds = Array.isArray(row.entity_ids) ? row.entity_ids.map(Number) : []; const entityRows = eIds.length > 0 - ? await sql`SELECT id, name, entity_type, metadata FROM entities WHERE id = ANY(${`{${eIds.join(',')}}`}::bigint[])` + ? await sql` + SELECT e.id, e.name, et.slug AS entity_type, e.metadata + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ANY(${`{${eIds.join(',')}}`}::bigint[]) + ` : []; // Fetch watcher name from version, slug from template (pre-consolidation) @@ -2069,14 +2079,14 @@ async function handleList( wr.created_at as watcher_run_created_at, wr.completed_at as watcher_run_completed_at, e.id as entity_id, - e.entity_type, + et.slug AS entity_type, e.name as entity_name, e.slug as entity_slug, e.organization_id, parent.id as parent_id, parent.name as parent_name, parent.slug as parent_slug, - parent.entity_type as parent_entity_type, + pet.slug as parent_entity_type, i.current_version_id, (SELECT COUNT(*) FROM watcher_windows iw WHERE iw.watcher_id = i.id) as windows_count `; @@ -2098,7 +2108,9 @@ async function handleList( query += ` FROM watchers i LEFT JOIN entities e ON e.id = ANY(i.entity_ids) + LEFT JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id LEFT JOIN watcher_versions cv ON i.current_version_id = cv.id ${buildLatestWatcherRunJoinSql('i', 'wr')} `; diff --git a/packages/owletto-backend/src/tools/get_content.ts b/packages/owletto-backend/src/tools/get_content.ts index 97e04af3d..d408088b0 100644 --- a/packages/owletto-backend/src/tools/get_content.ts +++ b/packages/owletto-backend/src/tools/get_content.ts @@ -553,14 +553,16 @@ export async function getContent( const entityResult = await sql` SELECT e.id, - e.entity_type, + et.slug AS entity_type, e.slug, e.parent_id, parent.slug as parent_slug, - parent.entity_type as parent_entity_type, + pet.slug as parent_entity_type, e.organization_id FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE e.id = ${entityId} `; @@ -1218,7 +1220,10 @@ export async function getContent( const uniqueEntityIds = Array.from(entityCountMap.keys()); const idList = `{${uniqueEntityIds.join(',')}}`; const entityRows = await sql` - SELECT id, name, entity_type FROM entities WHERE id = ANY(${idList}::int[]) + SELECT e.id, e.name, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ANY(${idList}::int[]) `; const entitySummary = entityRows @@ -1352,7 +1357,7 @@ async function handleWatcherMode( cv.condensation_prompt, cv.condensation_window_count, cv.version_sources, - (SELECT COALESCE(json_agg(json_build_object('id', e.id, 'name', e.name, 'type', e.entity_type)), '[]'::json) FROM entities e WHERE e.id = ANY(i.entity_ids)) as entities + (SELECT COALESCE(json_agg(json_build_object('id', e.id, 'name', e.name, 'type', et.slug)), '[]'::json) FROM entities e JOIN entity_types et ON et.id = e.entity_type_id WHERE e.id = ANY(i.entity_ids)) as entities FROM watchers i LEFT JOIN watcher_versions cv ON i.current_version_id = cv.id WHERE i.id = ${watcherId} diff --git a/packages/owletto-backend/src/tools/get_watchers.ts b/packages/owletto-backend/src/tools/get_watchers.ts index 52be4aacb..57bc107d1 100644 --- a/packages/owletto-backend/src/tools/get_watchers.ts +++ b/packages/owletto-backend/src/tools/get_watchers.ts @@ -319,11 +319,13 @@ export async function getWatcher( if (args.entity_id) { const entityCheck = await sql` - SELECT e.id, e.name, e.entity_type, e.slug, e.parent_id, - parent.slug as parent_slug, parent.entity_type as parent_entity_type, + SELECT e.id, e.name, et.slug AS entity_type, e.slug, e.parent_id, + parent.slug as parent_slug, pet.slug as parent_entity_type, e.organization_id FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE e.id = ${args.entity_id} `; @@ -336,12 +338,14 @@ export async function getWatcher( entitiesForTemplate = [{ name: info.entityName ?? '', type: info.entityType ?? '' }]; } else if (args.watcher_id) { const watcherEntityQuery = await sql` - SELECT e.id, e.name, e.entity_type, e.slug, e.parent_id, - parent.slug as parent_slug, parent.entity_type as parent_entity_type, + SELECT e.id, e.name, et.slug AS entity_type, e.slug, e.parent_id, + parent.slug as parent_slug, pet.slug as parent_entity_type, e.organization_id FROM watchers i JOIN entities e ON e.id = ANY(i.entity_ids) + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities parent ON e.parent_id = parent.id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id WHERE i.id = ${args.watcher_id} `; @@ -1085,16 +1089,17 @@ export async function getWatcher( SELECT CAST(e.id AS TEXT) as entity_id, e.name as entity_name, - e.entity_type, + et.slug AS entity_type, COUNT(DISTINCT f.id) as total_content, COUNT(DISTINCT c.connector_key) as active_connections, MAX(f.occurred_at) as latest_content_date FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN feeds fc ON e.id = ANY(fc.entity_ids) AND fc.deleted_at IS NULL LEFT JOIN connections c ON c.id = fc.connection_id AND c.organization_id = e.organization_id LEFT JOIN current_event_records f ON ${sql.unsafe(entityLinkMatchSql('e.id::bigint', 'f'))} WHERE e.id = ${contextEntityId} - GROUP BY e.id, e.name, e.entity_type + GROUP BY e.id, e.name, et.slug `; if (entityContextQuery.length > 0) { diff --git a/packages/owletto-backend/src/tools/resolve_path.ts b/packages/owletto-backend/src/tools/resolve_path.ts index 0c44d914a..cf663b05a 100644 --- a/packages/owletto-backend/src/tools/resolve_path.ts +++ b/packages/owletto-backend/src/tools/resolve_path.ts @@ -355,10 +355,11 @@ async function _resolvePath( if (!isLeaf) { // Lightweight query for intermediate path entities – no COUNT subqueries, no template joins const row = await simpleQuery(sql` - SELECT e.id, e.entity_type, e.slug, e.name, e.parent_id + SELECT e.id, et.slug AS entity_type, e.slug, e.name, e.parent_id FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.organization_id = ${workspace.id} - AND e.entity_type = ${segment.entity_type} + AND et.slug = ${segment.entity_type} AND e.slug = ${segment.slug} AND ( (${parentId}::bigint IS NULL AND e.parent_id IS NULL) @@ -389,7 +390,7 @@ async function _resolvePath( const row = await simpleQuery(sql` SELECT e.id, - e.entity_type, + et.slug AS entity_type, e.slug, e.name, e.parent_id, @@ -398,15 +399,13 @@ async function _resolvePath( COALESCE(vtv_entity.json_template, vtv_et.json_template) as json_template, COALESCE(vtv_entity.version, vtv_et.version) as json_template_version FROM entities e - LEFT JOIN entity_types et - ON et.slug = e.entity_type - AND et.organization_id = e.organization_id + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN view_template_versions vtv_entity ON vtv_entity.id = e.current_view_template_version_id LEFT JOIN view_template_versions vtv_et ON vtv_et.id = et.current_view_template_version_id WHERE e.organization_id = ${workspace.id} - AND e.entity_type = ${segment.entity_type} + AND et.slug = ${segment.entity_type} AND e.slug = ${segment.slug} AND ( (${parentId}::bigint IS NULL AND e.parent_id IS NULL) @@ -534,18 +533,20 @@ async function _resolvePath( // content_count is omitted to avoid expensive GIN index scans over the events table. const [childRows, siblingRows] = await Promise.all([ simpleQuery(sql` - SELECT e.id, e.entity_type, e.slug, e.name, + SELECT e.id, et.slug AS entity_type, e.slug, e.name, e.metadata::jsonb->>'market' as market FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.organization_id = ${workspace.id} AND e.parent_id = ${resolvedEntity.id} ORDER BY e.name ASC `), simpleQuery(sql` - SELECT e.id, e.entity_type, e.slug, e.name + SELECT e.id, et.slug AS entity_type, e.slug, e.name FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.organization_id = ${workspace.id} - AND e.entity_type = ${resolvedEntity.entity_type} + AND et.slug = ${resolvedEntity.entity_type} AND ( (${resolvedEntity.parent_id}::bigint IS NULL AND e.parent_id IS NULL) OR e.parent_id = ${resolvedEntity.parent_id} @@ -667,8 +668,7 @@ async function listEntityTypes( COUNT(e.id)::int AS entity_count FROM entity_types et LEFT JOIN entities e - ON e.organization_id = et.organization_id - AND e.entity_type = et.slug + ON e.entity_type_id = et.id WHERE et.deleted_at IS NULL AND et.organization_id = ${organizationId} GROUP BY et.id, et.slug, et.name, et.description, et.icon, et.color @@ -911,17 +911,19 @@ async function fetchRecentWatchers( e.name AS entity_name, e.slug AS entity_slug, parent.slug AS parent_slug, - parent.entity_type AS parent_entity_type, + pet.slug AS parent_entity_type, COALESCE(wwc.windows_count, 0)::int AS windows_count FROM scoped_watchers sw LEFT JOIN LATERAL ( - SELECT entity.id, entity.entity_type, entity.name, entity.slug, entity.parent_id + SELECT entity.id, et_ent.slug AS entity_type, entity.name, entity.slug, entity.parent_id FROM entities entity + JOIN entity_types et_ent ON et_ent.id = entity.entity_type_id WHERE entity.id = ANY(sw.entity_ids) ORDER BY entity.name ASC LIMIT 1 ) e ON TRUE LEFT JOIN entities parent ON parent.id = e.parent_id + LEFT JOIN entity_types pet ON pet.id = parent.entity_type_id LEFT JOIN watcher_window_counts wwc ON wwc.watcher_id = sw.id ORDER BY COALESCE(sw.updated_at, sw.created_at) DESC `); diff --git a/packages/owletto-backend/src/tools/save_content.ts b/packages/owletto-backend/src/tools/save_content.ts index b594031f5..4aa193769 100644 --- a/packages/owletto-backend/src/tools/save_content.ts +++ b/packages/owletto-backend/src/tools/save_content.ts @@ -183,11 +183,12 @@ export async function saveContent( SELECT e.id FROM entity_identities ei JOIN entities e ON e.id = ei.entity_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE ei.organization_id = ${ctx.organizationId} AND ei.namespace = 'auth_user_id' AND ei.identifier = ${authId} AND ei.deleted_at IS NULL - AND e.entity_type = '$member' + AND et.slug = '$member' AND e.deleted_at IS NULL LIMIT 1 `; @@ -202,11 +203,12 @@ export async function saveContent( SELECT e.id FROM entity_identities ei JOIN entities e ON e.id = ei.entity_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE ei.organization_id = ${ctx.organizationId} AND ei.namespace = 'email' AND ei.identifier = ${userEmail} AND ei.deleted_at IS NULL - AND e.entity_type = '$member' + AND et.slug = '$member' AND e.deleted_at IS NULL LIMIT 1 `; diff --git a/packages/owletto-backend/src/tools/search.ts b/packages/owletto-backend/src/tools/search.ts index dc2a79e26..1517e4f21 100644 --- a/packages/owletto-backend/src/tools/search.ts +++ b/packages/owletto-backend/src/tools/search.ts @@ -402,11 +402,12 @@ async function fetchTopEntitiesByType( ): Promise }>> { const sql = getDb(); const rows = await sql` - SELECT id, name, entity_type - FROM entities - WHERE organization_id = ${organizationId} - AND deleted_at IS NULL - ORDER BY (SELECT COUNT(*) FROM current_event_records ev WHERE ${sql.unsafe(entityLinkMatchSql('entities.id::bigint', 'ev'))}) DESC + SELECT e.id, e.name, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + ORDER BY (SELECT COUNT(*) FROM current_event_records ev WHERE ${sql.unsafe(entityLinkMatchSql('e.id::bigint', 'ev'))}) DESC LIMIT 30 `; @@ -428,8 +429,8 @@ async function fetchTopEntitiesByType( // ============================================ const ENTITY_SELECT_COLUMNS = ` - e.id, e.organization_id, e.name, e.entity_type, e.slug, e.metadata, e.parent_id, - pe.name as parent_name, pe.slug as parent_slug, pe.entity_type as parent_entity_type, + e.id, e.organization_id, e.name, et.slug AS entity_type, e.slug, e.metadata, e.parent_id, + pe.name as parent_name, pe.slug as parent_slug, pet.slug as parent_entity_type, COALESCE((SELECT COUNT(*) FROM current_event_records ev WHERE ${entityLinkMatchSql('e.id::bigint', 'ev')}), 0) as content_count, COALESCE(( SELECT COUNT(DISTINCT cn.connector_key) @@ -453,7 +454,9 @@ const ENTITY_SELECT_COLUMNS = ` const ENTITY_JOINS = ` FROM entities e - LEFT JOIN entities pe ON e.parent_id = pe.id`; + JOIN entity_types et ON et.id = e.entity_type_id + LEFT JOIN entities pe ON e.parent_id = pe.id + LEFT JOIN entity_types pet ON pet.id = pe.entity_type_id`; /** * Query entities by name with optional filters @@ -509,7 +512,7 @@ async function queryEntities( // Organization filter conditions.push(`e.organization_id = $${addParam(organizationId)}`); - if (args.entity_type) conditions.push(`e.entity_type = $${addParam(args.entity_type)}`); + if (args.entity_type) conditions.push(`et.slug = $${addParam(args.entity_type)}`); if (args.parent_id) conditions.push(`e.parent_id = $${addParam(args.parent_id)}`); if (args.category) conditions.push(`e.metadata::jsonb->>'category' = $${addParam(args.category)}`); @@ -640,13 +643,14 @@ async function formatEntityResult( SELECT e.id, e.name, - e.entity_type, + et.slug AS entity_type, e.metadata::jsonb->>'market' as market, COALESCE( (SELECT COUNT(*) FROM current_event_records WHERE e.id = ANY(entity_ids)), 0 ) as content_count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.parent_id = ${primaryEntity.id} ORDER BY e.created_at DESC `; diff --git a/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts b/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts index 08e9bd2db..5abf0426e 100644 --- a/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts +++ b/packages/owletto-backend/src/utils/__tests__/entity-link-upsert.test.ts @@ -84,8 +84,9 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const entities = await sql` - SELECT id, name, metadata FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT e.id, e.name, e.metadata FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entities).toHaveLength(1); expect(entities[0].name).toBe('Alex'); @@ -107,8 +108,12 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const [{ id: entityId }] = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'Alex', 'member-seed', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'Alex', 'member-seed', '{}'::jsonb, ${user.id} + ) RETURNING id `; await sql` @@ -138,8 +143,9 @@ describe('applyEntityLinks', () => { }); const entityCount = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entityCount[0].count).toBe('1'); @@ -156,13 +162,21 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const entA = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'A', 'member-a', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'A', 'member-a', '{}'::jsonb, ${user.id} + ) RETURNING id `; const entB = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'B', 'member-b', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'B', 'member-b', '{}'::jsonb, ${user.id} + ) RETURNING id `; await sql` @@ -194,8 +208,9 @@ describe('applyEntityLinks', () => { // No new entity created, no new identifiers accreted to either side. const entities = await sql<{ count: string }[]>` - SELECT COUNT(*)::text AS count FROM entities - WHERE organization_id = ${org.id} AND entity_type = '$member' AND deleted_at IS NULL + SELECT COUNT(*)::text AS count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${org.id} AND et.slug = '$member' AND e.deleted_at IS NULL `; expect(entities[0].count).toBe('2'); @@ -247,8 +262,12 @@ describe('applyEntityLinks', () => { const sql = getTestDb(); const [{ id: entityId }] = await sql<{ id: number | string }[]>` - INSERT INTO entities (organization_id, entity_type, name, slug, metadata, created_by) - VALUES (${org.id}, '$member', 'Alex', 'member-alex', '{}'::jsonb, ${user.id}) + INSERT INTO entities (organization_id, entity_type_id, name, slug, metadata, created_by) + VALUES ( + ${org.id}, + (SELECT id FROM entity_types WHERE slug = '$member' AND organization_id = ${org.id} AND deleted_at IS NULL), + 'Alex', 'member-alex', '{}'::jsonb, ${user.id} + ) RETURNING id `; await sql` diff --git a/packages/owletto-backend/src/utils/auto-linker.ts b/packages/owletto-backend/src/utils/auto-linker.ts index b6a7ec8ef..94f2d40a5 100644 --- a/packages/owletto-backend/src/utils/auto-linker.ts +++ b/packages/owletto-backend/src/utils/auto-linker.ts @@ -35,11 +35,13 @@ async function getOrgEntities(organizationId: string): Promise= ${MIN_NAME_LENGTH} - ORDER BY length(name) DESC + SELECT e.id, e.name, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + AND length(e.name) >= ${MIN_NAME_LENGTH} + ORDER BY length(e.name) DESC `; const entities = rows.map((r) => ({ diff --git a/packages/owletto-backend/src/utils/entity-link-upsert.ts b/packages/owletto-backend/src/utils/entity-link-upsert.ts index f8592820c..ca86ca472 100644 --- a/packages/owletto-backend/src/utils/entity-link-upsert.ts +++ b/packages/owletto-backend/src/utils/entity-link-upsert.ts @@ -178,10 +178,11 @@ async function lookupMatches(params: { SELECT ei.entity_id, ei.namespace, ei.identifier FROM entity_identities ei JOIN entities e ON e.id = ei.entity_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE ei.organization_id = ${params.orgId} AND ei.deleted_at IS NULL AND e.deleted_at IS NULL - AND e.entity_type = ${params.entityType} + AND et.slug = ${params.entityType} AND (ei.namespace, ei.identifier) IN ( SELECT ns, ident FROM unnest(${pgTextArray(namespaces)}::text[], ${pgTextArray(identifiers)}::text[]) AS u(ns, ident) ) @@ -211,6 +212,23 @@ async function createEntityWithIdentities(params: { const metadata: Record = {}; for (const [key, value] of params.traits) metadata[key] = value; + // Resolve entity_type slug to FK on entity_types(id). + const typeRow = await sql<{ id: number }>` + SELECT id FROM entity_types + WHERE slug = ${params.entityType} + AND organization_id = ${params.orgId} + AND deleted_at IS NULL + LIMIT 1 + `; + if (typeRow.length === 0) { + logger.warn( + { entityType: params.entityType, orgId: params.orgId }, + 'entity create failed: unknown entity type' + ); + return null; + } + const entityTypeId = typeRow[0].id; + // Try a few slug variants to defuse improbable random collisions. let entityId: number | null = null; for (let attempt = 0; attempt < 3 && entityId === null; attempt++) { @@ -218,11 +236,11 @@ async function createEntityWithIdentities(params: { try { const rows = await sql<{ id: number | string }>` INSERT INTO entities ( - organization_id, entity_type, name, slug, metadata, + organization_id, entity_type_id, name, slug, metadata, created_by, created_at, updated_at ) VALUES ( - ${params.orgId}, ${params.entityType}, ${name}, ${slug}, + ${params.orgId}, ${entityTypeId}, ${name}, ${slug}, ${sql.json(metadata)}, ${params.creatorUserId}, current_timestamp, current_timestamp ) diff --git a/packages/owletto-backend/src/utils/entity-management.ts b/packages/owletto-backend/src/utils/entity-management.ts index f2a1f0c1b..c2e06dade 100644 --- a/packages/owletto-backend/src/utils/entity-management.ts +++ b/packages/owletto-backend/src/utils/entity-management.ts @@ -232,21 +232,21 @@ export async function createEntity( const sql = getDb(); - // Validate entity type exists in entity_types table - const typeCheck = await sql.unsafe( - `SELECT id FROM entity_types - WHERE slug = $1 - AND deleted_at IS NULL - AND organization_id = $2 - LIMIT 1`, - [data.entity_type, data.organization_id] - ); - if (typeCheck.length === 0) { + // Resolve entity_type slug to FK on entity_types(id). + const typeRow = await sql<{ id: number }>` + SELECT id FROM entity_types + WHERE slug = ${data.entity_type} + AND deleted_at IS NULL + AND organization_id = ${data.organization_id} + LIMIT 1 + `; + if (typeRow.length === 0) { throw new ToolUserError( `Unknown entity type '${data.entity_type}'. Use manage_entity_schema(schema_type="entity_type", action="list") to list available types or create a custom type first.`, 400 ); } + const entityTypeId = typeRow[0].id; // Generate slug from name if not provided const slug = data.slug || generateSlug(data.name); @@ -265,22 +265,24 @@ export async function createEntity( const contentHash = data.content_hash || null; try { - const result = await sql` + const result = await sql>` INSERT INTO entities ( - organization_id, entity_type, name, slug, parent_id, metadata, enabled_classifiers, created_by, content, embedding, content_hash, created_at, updated_at + organization_id, entity_type_id, name, slug, parent_id, metadata, enabled_classifiers, created_by, content, embedding, content_hash, created_at, updated_at ) VALUES ( - ${data.organization_id}, ${data.entity_type}, ${data.name.trim()}, ${slug}, ${data.parent_id || null}, + ${data.organization_id}, ${entityTypeId}, ${data.name.trim()}, ${slug}, ${data.parent_id || null}, ${sql.json(metadata)}, ${data.enabled_classifiers || null}, ${createdBy}, ${contentValue}, ${embeddingLiteral}::vector, ${contentHash}, current_timestamp, current_timestamp ) - RETURNING id, entity_type, name, slug, parent_id, metadata, created_at + RETURNING id, name, slug, parent_id, metadata, created_at `; if (result.length === 0) { throw new Error('Failed to create entity'); } - const created = result[0]; + // The validator above already resolved data.entity_type → entityTypeId. + // Pass the slug back through directly rather than JOIN-ing on every insert. + const created: CreatedEntity = { ...result[0], entity_type: data.entity_type }; // Run afterCreate hook if (!opts?.skipHooks && opts?.hookContext) { @@ -376,9 +378,10 @@ export async function updateEntity( `; const result = await sql` - SELECT id, entity_type, name, slug, parent_id, metadata, created_at - FROM entities - WHERE id = ${entityId} + SELECT e.id, et.slug AS entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ${entityId} LIMIT 1 `; @@ -403,9 +406,9 @@ export async function getEntity( const result = await sql` SELECT - e.id, e.entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, + e.id, et.slug AS entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, e.current_view_template_version_id, - pe.name as parent_name, pe.slug as parent_slug, pe.entity_type as parent_entity_type, + pe.name as parent_name, pe.slug as parent_slug, pet.slug as parent_entity_type, (SELECT COUNT(*) FROM current_event_records ev WHERE ${sql.unsafe(entityLinkMatchSql('e.id::bigint', 'ev'))}) as total_content, ( SELECT COUNT(DISTINCT c.connector_key) @@ -418,7 +421,9 @@ export async function getEntity( (SELECT COUNT(*) FROM watchers i WHERE e.id = ANY(i.entity_ids)) as watchers_count, (SELECT COUNT(*) FROM entities c WHERE c.parent_id = e.id) as children_count FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities pe ON e.parent_id = pe.id + LEFT JOIN entity_types pet ON pet.id = pe.entity_type_id WHERE e.id = ${entityId} AND e.organization_id = ${ctx.organizationId} AND e.deleted_at IS NULL @@ -448,7 +453,10 @@ export async function deleteEntity( // Run beforeDelete hook if (!opts?.skipHooks) { const entityRow = await sql` - SELECT entity_type, metadata FROM entities WHERE id = ${entityId} AND deleted_at IS NULL + SELECT et.slug AS entity_type, e.metadata + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id = ${entityId} AND e.deleted_at IS NULL `; if (entityRow.length > 0) { const hooks = getEntityHooks(entityRow[0].entity_type as string); @@ -655,7 +663,7 @@ export async function listEntities( params.push(ctx.organizationId); if (filters.entity_type) { - conditions.push(`e.entity_type = $${paramIdx++}`); + conditions.push(`et.slug = $${paramIdx++}`); params.push(filters.entity_type); } @@ -707,7 +715,9 @@ export async function listEntities( const baseQuery = ` FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id LEFT JOIN entities pe ON e.parent_id = pe.id + LEFT JOIN entity_types pet ON pet.id = pe.entity_type_id LEFT JOIN LATERAL (SELECT COUNT(*) as cnt FROM current_event_records ev WHERE ${entityLinkMatchSql('e.id::bigint', 'ev')}) tc ON true LEFT JOIN LATERAL ( SELECT COUNT(DISTINCT c.connector_key) as cnt @@ -729,12 +739,12 @@ export async function listEntities( const result = await sql.unsafe( `SELECT - e.id, e.entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, + e.id, et.slug AS entity_type, e.name, e.slug, e.parent_id, e.metadata, e.created_at, COALESCE(tc.cnt, 0) as total_content, COALESCE(ac.cnt, 0) as active_connections, COALESCE(ic.cnt, 0) as watchers_count, COALESCE(cc.cnt, 0) as children_count, - pe.name as parent_name, pe.slug as parent_slug, pe.entity_type as parent_entity_type + pe.name as parent_name, pe.slug as parent_slug, pet.slug as parent_entity_type ${baseQuery} ORDER BY ${orderBy} LIMIT ${limit + 1} @@ -786,12 +796,14 @@ export async function batchLoadRelationships( r.from_entity_id, r.to_entity_id, rt.slug AS relationship_type_slug, - fe.id AS from_id, fe.name AS from_name, fe.slug AS from_slug, fe.entity_type AS from_entity_type, - te.id AS to_id, te.name AS to_name, te.slug AS to_slug, te.entity_type AS to_entity_type + fe.id AS from_id, fe.name AS from_name, fe.slug AS from_slug, fet.slug AS from_entity_type, + te.id AS to_id, te.name AS to_name, te.slug AS to_slug, tet.slug AS to_entity_type FROM entity_relationships r JOIN entity_relationship_types rt ON r.relationship_type_id = rt.id LEFT JOIN entities fe ON r.from_entity_id = fe.id + LEFT JOIN entity_types fet ON fet.id = fe.entity_type_id LEFT JOIN entities te ON r.to_entity_id = te.id + LEFT JOIN entity_types tet ON tet.id = te.entity_type_id WHERE r.organization_id = ${organizationId} AND r.deleted_at IS NULL AND rt.slug = ANY(${typeSlugs}::text[]) diff --git a/packages/owletto-backend/src/utils/event-kind-validation.ts b/packages/owletto-backend/src/utils/event-kind-validation.ts index d336be342..5492fc3b8 100644 --- a/packages/owletto-backend/src/utils/event-kind-validation.ts +++ b/packages/owletto-backend/src/utils/event-kind-validation.ts @@ -134,7 +134,7 @@ async function getEntityTypeEventKinds( const rows = await sql` SELECT et.event_kinds FROM entities e - JOIN entity_types et ON et.slug = e.entity_type AND et.organization_id = e.organization_id + JOIN entity_types et ON et.id = e.entity_type_id WHERE e.id = ${entityId} AND e.organization_id = ${orgId} AND e.deleted_at IS NULL diff --git a/packages/owletto-backend/src/utils/execute-data-sources.ts b/packages/owletto-backend/src/utils/execute-data-sources.ts index 71c18dfa9..0f13c6386 100644 --- a/packages/owletto-backend/src/utils/execute-data-sources.ts +++ b/packages/owletto-backend/src/utils/execute-data-sources.ts @@ -215,13 +215,33 @@ function buildScopedQuery( return buildColumnList(defs, alias); }; + // Build the SELECT list for the entities CTE, where entity_type is now a + // derived column from a JOIN to entity_types (et.slug AS entity_type). + const selEntitiesJoined = (entityAlias: string, typeAlias: string): string => { + const defs = sc?.get('entities'); + if (!defs) return `${entityAlias}.*, ${typeAlias}.slug AS entity_type`; + return defs + .map((c) => { + if (c.name === 'entity_type') return `${typeAlias}.slug AS "entity_type"`; + if (c.expr) { + const prefixed = c.expr.replace(/^(\w+)/, `${entityAlias}.$1`); + return `${prefixed} as "${c.name}"`; + } + return `${entityAlias}."${c.name}"`; + }) + .join(', '); + }; + for (const table of tableRefs) { // Escape double quotes in table name for safe identifier quoting const safeName = table.replace(/"/g, '""'); if (table === 'entities') { ctes.push( - `"${safeName}" AS (SELECT ${sel(table)} FROM public.entities WHERE organization_id = ${orgP})` + `"${safeName}" AS (SELECT ${selEntitiesJoined('e', 'et')} ` + + `FROM public.entities e ` + + `JOIN public.entity_types et ON et.id = e.entity_type_id ` + + `WHERE e.organization_id = ${orgP})` ); } else if (table === 'events') { let eventsCte = @@ -318,7 +338,10 @@ function buildScopedQuery( idx++; params.push(table); ctes.push( - `"${safeName}" AS (SELECT ${sel('entities')} FROM public.entities WHERE organization_id = ${orgP} AND entity_type = $${idx})` + `"${safeName}" AS (SELECT ${selEntitiesJoined('e', 'et')} ` + + `FROM public.entities e ` + + `JOIN public.entity_types et ON et.id = e.entity_type_id ` + + `WHERE e.organization_id = ${orgP} AND et.slug = $${idx})` ); } } diff --git a/packages/owletto-backend/src/utils/member-entity.ts b/packages/owletto-backend/src/utils/member-entity.ts index ef5ba2ddd..c1cfba7c2 100644 --- a/packages/owletto-backend/src/utils/member-entity.ts +++ b/packages/owletto-backend/src/utils/member-entity.ts @@ -52,11 +52,13 @@ export async function ensureMemberEntity(params: EnsureMemberEntityParams): Prom // Check if a $member entity with this email already exists const existing = await sql.unsafe( - `SELECT id FROM entities - WHERE entity_type = '$member' - AND organization_id = $1 - AND metadata->>$2 = $3 - AND deleted_at IS NULL + `SELECT e.id + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = '$member' + AND e.organization_id = $1 + AND e.metadata->>$2 = $3 + AND e.deleted_at IS NULL LIMIT 1`, [params.organizationId, emailField, params.email] ); @@ -95,7 +97,10 @@ export async function updateMemberEntityStatus( UPDATE entities SET metadata = jsonb_set(metadata, '{status}', to_jsonb(${status}::text)), updated_at = current_timestamp - WHERE entity_type = '$member' + WHERE entity_type_id = ( + SELECT id FROM entity_types + WHERE slug = '$member' AND organization_id = ${organizationId} AND deleted_at IS NULL + ) AND organization_id = ${organizationId} AND metadata->>${emailField} = ${email} AND deleted_at IS NULL @@ -111,11 +116,13 @@ export async function updateMemberEntityAccess( const { emailField } = await resolveMemberSchemaFields(organizationId); const sql = getDb(); const rows = await sql.unsafe<{ id: number; metadata: Record }>( - `SELECT id, metadata FROM entities - WHERE entity_type = '$member' - AND organization_id = $1 - AND metadata->>$2 = $3 - AND deleted_at IS NULL + `SELECT e.id, e.metadata + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE et.slug = '$member' + AND e.organization_id = $1 + AND e.metadata->>$2 = $3 + AND e.deleted_at IS NULL LIMIT 1`, [organizationId, emailField, email] ); @@ -143,7 +150,10 @@ export async function deleteMemberEntity(organizationId: string, email: string): await sql.unsafe( `UPDATE entities SET deleted_at = current_timestamp, updated_at = current_timestamp - WHERE entity_type = '$member' + WHERE entity_type_id = ( + SELECT id FROM entity_types + WHERE slug = '$member' AND organization_id = $1 AND deleted_at IS NULL + ) AND organization_id = $1 AND metadata->>$2 = $3 AND deleted_at IS NULL`, diff --git a/packages/owletto-backend/src/utils/relationship-validation.ts b/packages/owletto-backend/src/utils/relationship-validation.ts index db53479ec..ac2fbe1ba 100644 --- a/packages/owletto-backend/src/utils/relationship-validation.ts +++ b/packages/owletto-backend/src/utils/relationship-validation.ts @@ -115,9 +115,10 @@ export async function validateTypeRule( // Get entity types for both entities const entityRows = await sql` - SELECT id, entity_type - FROM entities - WHERE id IN (${fromEntityId}, ${toEntityId}) + SELECT e.id, et.slug AS entity_type + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.id IN (${fromEntityId}, ${toEntityId}) `; const fromEntityType = String(entityRows.find((r) => Number(r.id) === fromEntityId)?.entity_type); const toEntityType = String(entityRows.find((r) => Number(r.id) === toEntityId)?.entity_type); diff --git a/packages/owletto-backend/src/utils/table-schema.ts b/packages/owletto-backend/src/utils/table-schema.ts index bccac23e3..49f1840be 100644 --- a/packages/owletto-backend/src/utils/table-schema.ts +++ b/packages/owletto-backend/src/utils/table-schema.ts @@ -26,6 +26,8 @@ function cols(...names: string[]): ColumnDef[] { export const QUERYABLE_SCHEMA = { tables: [ // entities (excludes: embedding, content_tsv, content_hash) + // entity_type is exposed as a derived column — the CTE JOINs entity_types + // and aliases et.slug AS entity_type, so user queries can keep referencing it. { name: 'entities', columns: cols( diff --git a/packages/owletto-backend/src/utils/workspace-instructions.ts b/packages/owletto-backend/src/utils/workspace-instructions.ts index 6005f3a75..1595e7605 100644 --- a/packages/owletto-backend/src/utils/workspace-instructions.ts +++ b/packages/owletto-backend/src/utils/workspace-instructions.ts @@ -22,10 +22,12 @@ export async function buildWorkspaceInstructions(organizationId: string): Promis [organizationId] ), sql` - SELECT entity_type, COUNT(*)::int as entity_count - FROM entities - WHERE organization_id = ${organizationId} - GROUP BY entity_type + SELECT et.slug AS entity_type, COUNT(*)::int as entity_count + FROM entities e + JOIN entity_types et ON et.id = e.entity_type_id + WHERE e.organization_id = ${organizationId} + AND e.deleted_at IS NULL + GROUP BY et.slug `, sql.unsafe( `SELECT rt.slug, rt.name, rt.is_symmetric, inv.slug as inverse_type_slug,