From 248257b8756701bd9d20439574bfa3b1caac6ad2 Mon Sep 17 00:00:00 2001 From: Gabriel Birman <25272206+gbirman@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:07:14 +0000 Subject: [PATCH 1/3] feat(messages): idempotent import of legacy document comments Adds comment_import, a feature-gated binary in macro_db_migrator that projects the legacy Comment and Thread rows onto comms_messages and comms_message_threads one batch of documents at a time while the application stays live. Every run upserts only rows whose projection differs, so reruns are no-ops and later legacy writes arrive as deltas. Mapping ids are allocated once as UUIDv7 in the same transaction as the rows they point at, which lets a normal migration add the deferred foreign keys from the mapping tables to the message tables. Also remaps notification metadata and saved PDF payload comment ids, fills root_id on the PDF anchor tables, marks empty legacy threads deleted, protects newer message store writes from stale legacy rows, and adds LEGACY_COMMENT_WRITES_ENABLED to the document storage service so the final pass can run against a frozen source. --- ...bdedef8ac3d00b378999c7370d1b6f8ec22be.json | 22 + ...54f5e71c055fd4a36eba60c158d663d8d65b6.json | 16 + ...6c86de0e00f6bce372381114a12453714a39d.json | 24 + ...9c90499802d0ebe46f7a5b2e5d0f958c12364.json | 28 + ...6bf41bc489612d22ae37c8bf52989ba8e062a.json | 14 + ...0b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json | 23 + ...02c76b9e44ee49efec6c4afc9ea620cb27f2f.json | 35 + ...8fa9ab8a3df340502c7e416aa8d445e5f5b86.json | 16 + ...15423ead933fb94bbd3dad5e6e5e7ff59ff53.json | 22 + ...aa081cc95ce28236cf6a7aa97b37556caa98e.json | 35 + ...c5a0cd749cc32100c3d95c0cc6792e697e8a9.json | 22 + ...67643329cd92029864abda17d5376dc999739.json | 15 + ...f2a9dde0e7f00dd58622870b2e7b3e49b79f2.json | 22 + ...d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json | 20 + ...eb565faee20148c38b8ca42582c02628af0e7.json | 16 + ...621edab609f4eaba0ce9a4da11743cc2b1482.json | 15 + ...166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json | 14 + ...659623a2e31379ba795b68a366b2d6439a516.json | 22 + ...2932436bf3ce8a810b8a868655f399e0fc3f5.json | 22 + ...49fa70745c460e53ec316be184a309f492754.json | 14 + ...47b8591213be9dc848d874b3b3f2a78226669.json | 22 + ...48a7b7daf2813c73521fbb8691d4f34ca28ea.json | 20 + ...11fc307ac363e4d1701439b0bc6ad59e29d16.json | 23 + ...b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json | 34 + ...43f58913909a51b4f5f6ef057f8cccdd8eab7.json | 28 + ...fe3aa835c20b17993e9e8cdabfe20ed5efa34.json | 14 + ...c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json | 20 + Cargo.lock | 10 + ...17190902_messages_mapping_foreign_keys.sql | 20 + crates/macro_db_migrator/Cargo.toml | 29 + .../src/bin/comment_import.rs | 45 + .../src/bin/comment_import/legacy_fixture.sql | 46 + .../src/bin/comment_import/runner.rs | 766 +++++++++++++++++ .../src/bin/comment_import/summary.rs | 169 ++++ .../src/bin/comment_import/test.rs | 793 ++++++++++++++++++ .../src/outbound/pg_message_repo/test.rs | 66 ++ docs/legacy-comment-import.md | 138 +++ .../src/api/annotations.rs | 28 +- .../src/api/annotations/write_switch.rs | 39 + .../src/api/annotations/write_switch/test.rs | 37 + .../document_storage_service/src/config.rs | 8 + 41 files changed, 2764 insertions(+), 8 deletions(-) create mode 100644 .sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json create mode 100644 .sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json create mode 100644 .sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json create mode 100644 .sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json create mode 100644 .sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json create mode 100644 .sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json create mode 100644 .sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json create mode 100644 .sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json create mode 100644 .sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json create mode 100644 .sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json create mode 100644 .sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json create mode 100644 .sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json create mode 100644 .sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json create mode 100644 .sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json create mode 100644 .sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json create mode 100644 .sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json create mode 100644 .sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json create mode 100644 .sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json create mode 100644 .sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json create mode 100644 .sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json create mode 100644 .sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json create mode 100644 .sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json create mode 100644 .sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json create mode 100644 .sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json create mode 100644 .sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json create mode 100644 .sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json create mode 100644 .sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json create mode 100644 crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql create mode 100644 crates/macro_db_migrator/src/bin/comment_import.rs create mode 100644 crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql create mode 100644 crates/macro_db_migrator/src/bin/comment_import/runner.rs create mode 100644 crates/macro_db_migrator/src/bin/comment_import/summary.rs create mode 100644 crates/macro_db_migrator/src/bin/comment_import/test.rs create mode 100644 docs/legacy-comment-import.md create mode 100644 services/document_storage_service/src/api/annotations/write_switch.rs create mode 100644 services/document_storage_service/src/api/annotations/write_switch/test.rs diff --git a/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json b/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json new file mode 100644 index 00000000000..94d118fc91e --- /dev/null +++ b/.sqlx/query-038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO comms_messages (\n id, parent_entity_type, parent_entity_id, sender_id, content,\n created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, '',\n t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n COALESCE(t.\"deletedAt\", t.\"updatedAt\")\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n ON CONFLICT (id) DO UPDATE SET\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE comms_messages.updated_at <= EXCLUDED.updated_at\n AND (comms_messages.updated_at, comms_messages.deleted_at)\n IS DISTINCT FROM (EXCLUDED.updated_at, EXCLUDED.deleted_at)\n RETURNING (xmax = 0) AS \"inserted!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inserted!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "038db5b91138c574e5927816834bdedef8ac3d00b378999c7370d1b6f8ec22be" +} diff --git a/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json b/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json new file mode 100644 index 00000000000..605567a0340 --- /dev/null +++ b/.sqlx/query-099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id)\n SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "UuidArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "099de7baaa6a5bbb26a23719ae554f5e71c055fd4a36eba60c158d663d8d65b6" +} diff --git a/.sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json b/.sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json new file mode 100644 index 00000000000..09728c3896a --- /dev/null +++ b/.sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE t.\"documentId\" = ANY($1) AND t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' IS NOT NULL\n AND t.metadata->>'markId' !~ $2\n AND t.metadata->>'markId' NOT LIKE $3\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d" +} diff --git a/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json b/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json new file mode 100644 index 00000000000..c16520dedf3 --- /dev/null +++ b/.sqlx/query-12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT comment_id, message_id FROM migrated_comment_id WHERE document_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "comment_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "message_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "12eba8d33668ec91d11bf8787de9c90499802d0ebe46f7a5b2e5d0f958c12364" +} diff --git a/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json b/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json new file mode 100644 index 00000000000..261ad9c71a0 --- /dev/null +++ b/.sqlx/query-180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE \"PdfPlaceableCommentAnchor\" a SET root_id = tm.root_id\n FROM migrated_comment_thread_id tm\n WHERE a.\"threadId\" = tm.thread_id AND a.\"documentId\" = ANY($1)\n AND a.root_id IS DISTINCT FROM tm.root_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "180a9c059064bca1eab390de5c76bf41bc489612d22ae37c8bf52989ba8e062a" +} diff --git a/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json b/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json new file mode 100644 index 00000000000..9cf00258a3c --- /dev/null +++ b/.sqlx/query-27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM notification n\n WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)\n AND n.notification_event_type = ANY($2)\n AND (n.metadata->>'commentId' ~ '^[0-9]+$' OR n.metadata->>'threadId' ~ '^[0-9]+$')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "27dba71899c33b28e6f3daf31030b8e9f609ee7e6b0a62bba74ac168ac76f8cd" +} diff --git a/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json b/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json new file mode 100644 index 00000000000..7fbf5c1431e --- /dev/null +++ b/.sqlx/query-3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT m.id, i.\"documentId\" AS document_id, m.\"modificationData\" AS payload\n FROM \"DocumentInstanceModificationData\" m\n JOIN \"DocumentInstance\" i ON i.id = m.\"documentInstanceId\"\n JOIN \"Document\" d ON d.id = i.\"documentId\"\n WHERE i.\"documentId\" = ANY($1) AND d.\"fileType\" = 'pdf'\n AND jsonb_path_exists(m.\"modificationData\", $2::text::jsonpath)\n ORDER BY m.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "document_id", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "3d92901e27e9ca5ff9f78b9f09e02c76b9e44ee49efec6c4afc9ea620cb27f2f" +} diff --git a/.sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json b/.sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json new file mode 100644 index 00000000000..f710cda86bb --- /dev/null +++ b/.sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO comms_message_threads (\n root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,\n import_metadata, created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, t.resolved,\n CASE\n WHEN pa.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)\n WHEN ph.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)\n WHEN t.metadata->>'markId' ~ $3\n THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)\n END,\n t.metadata, t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE\n WHEN t.\"deletedAt\" IS NOT NULL THEN t.\"deletedAt\" AT TIME ZONE 'UTC'\n WHEN NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n THEN t.\"updatedAt\" AT TIME ZONE 'UTC'\n END\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfPlaceableCommentAnchor\" a WHERE a.\"threadId\" = t.id\n ORDER BY a.uuid LIMIT 1\n ) pa ON true\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfHighlightAnchor\" a\n WHERE a.\"threadId\" = t.id AND a.\"deletedAt\" IS NULL\n ORDER BY a.uuid LIMIT 1\n ) ph ON true\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (root_id) DO UPDATE SET\n user_id = EXCLUDED.user_id,\n resolved = EXCLUDED.resolved,\n anchor = EXCLUDED.anchor,\n import_metadata = EXCLUDED.import_metadata,\n created_at = EXCLUDED.created_at,\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE (comms_message_threads.root_id = ANY($2)\n OR comms_message_threads.updated_at <= EXCLUDED.updated_at)\n AND (comms_message_threads.user_id, comms_message_threads.resolved,\n comms_message_threads.anchor, comms_message_threads.import_metadata,\n comms_message_threads.created_at, comms_message_threads.updated_at,\n comms_message_threads.deleted_at)\n IS DISTINCT FROM\n (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,\n EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86" +} diff --git a/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json b/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json new file mode 100644 index 00000000000..733b7ba9873 --- /dev/null +++ b/.sqlx/query-4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_unlock($1) AS \"released!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "released!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4027823662c88d28cb2a95792b615423ead933fb94bbd3dad5e6e5e7ff59ff53" +} diff --git a/.sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json b/.sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json new file mode 100644 index 00000000000..6f91ea852a0 --- /dev/null +++ b/.sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT t.\"documentId\" AS document_id, t.metadata->>'markId' AS \"mark_id!\",\n array_agg(t.id ORDER BY t.id) AS \"thread_ids!\"\n FROM \"Thread\" t\n WHERE t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' ~ $1\n AND EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)\n GROUP BY 1, 2 HAVING count(*) > 1\n ORDER BY 1, 2 LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "document_id", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "mark_id!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "thread_ids!", + "type_info": "Int8Array" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e" +} diff --git a/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json b/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json new file mode 100644 index 00000000000..38304c51cb2 --- /dev/null +++ b/.sqlx/query-576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n JOIN LATERAL (\n SELECT cm.message_id FROM \"Comment\" c\n JOIN migrated_comment_id cm ON cm.comment_id = c.id\n WHERE c.\"threadId\" = t.id\n ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id LIMIT 1\n ) first ON true\n WHERE t.\"documentId\" = ANY($1) AND first.message_id <> tm.root_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "576c0d1c1c043d4418c8f137138c5a0cd749cc32100c3d95c0cc6792e697e8a9" +} diff --git a/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json b/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json new file mode 100644 index 00000000000..a0d957a6c96 --- /dev/null +++ b/.sqlx/query-624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE \"DocumentInstanceModificationData\" SET \"modificationData\" = $1 WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "624ac87b24d542ae21dabeb58d067643329cd92029864abda17d5376dc999739" +} diff --git a/.sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json b/.sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json new file mode 100644 index 00000000000..3fda437afbe --- /dev/null +++ b/.sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT t.id FROM \"Thread\" t\n LEFT JOIN \"PdfPlaceableCommentAnchor\" p ON p.\"threadId\" = t.id\n LEFT JOIN \"PdfHighlightAnchor\" h ON h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL\n WHERE t.\"deletedAt\" IS NULL\n GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1\n ORDER BY t.id LIMIT $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2" +} diff --git a/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json b/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json new file mode 100644 index 00000000000..57c0fe1328d --- /dev/null +++ b/.sqlx/query-72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM \"Comment\" c\n WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "72c6260a4c0cdfb61d8273040c5d79d6c615c3e1dcbb4bbfabc9a61e2cead4cd" +} diff --git a/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json b/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json new file mode 100644 index 00000000000..c2e34c72c14 --- /dev/null +++ b/.sqlx/query-74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO migrated_comment_id (comment_id, message_id, document_id)\n SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "UuidArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "74275afbf7e9f3348efbbd7d8c7eb565faee20148c38b8ca42582c02628af0e7" +} diff --git a/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json b/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json new file mode 100644 index 00000000000..fb2702be6c2 --- /dev/null +++ b/.sqlx/query-76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE notification n SET metadata = jsonb_set(\n jsonb_set(n.metadata, '{commentId}', to_jsonb(cm.message_id::text)),\n '{threadId}', to_jsonb(tm.root_id::text)\n )\n FROM migrated_comment_id cm, migrated_comment_thread_id tm\n WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1)\n AND n.notification_event_type = ANY($2)\n AND cm.document_id = n.event_item_id AND tm.document_id = n.event_item_id\n AND n.metadata->>'commentId' = cm.comment_id::text\n AND n.metadata->>'threadId' = tm.thread_id::text", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "76f3e71eaea836800a3e5857a00621edab609f4eaba0ce9a4da11743cc2b1482" +} diff --git a/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json b/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json new file mode 100644 index 00000000000..3ef0c292413 --- /dev/null +++ b/.sqlx/query-7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE \"PdfHighlightAnchor\" a SET root_id = tm.root_id\n FROM migrated_comment_thread_id tm\n WHERE a.\"threadId\" = tm.thread_id AND a.\"documentId\" = ANY($1)\n AND a.root_id IS DISTINCT FROM tm.root_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "7a590cadb532aa035907f3c2889166dd7c5cea5d61ab9e7fedf9f8a5186295e7" +} diff --git a/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json b/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json new file mode 100644 index 00000000000..c7196a52c71 --- /dev/null +++ b/.sqlx/query-8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_lock($1) AS \"acquired!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "acquired!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8da419734f41296de7dd848d4b2659623a2e31379ba795b68a366b2d6439a516" +} diff --git a/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json b/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json new file mode 100644 index 00000000000..baa475902dc --- /dev/null +++ b/.sqlx/query-99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE comms_message_threads s SET deleted_at = now(), updated_at = now()\n FROM migrated_comment_thread_id tm\n WHERE tm.root_id = s.root_id AND tm.document_id = ANY($1) AND s.deleted_at IS NULL\n AND NOT EXISTS (SELECT 1 FROM \"Thread\" t WHERE t.id = tm.thread_id)\n RETURNING s.root_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "root_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "99086fa5c096f71daeb129691382932436bf3ce8a810b8a868655f399e0fc3f5" +} diff --git a/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json b/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json new file mode 100644 index 00000000000..102b4ff2699 --- /dev/null +++ b/.sqlx/query-a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE comms_messages SET deleted_at = now(), updated_at = now()\n WHERE (id = ANY($1) OR thread_id = ANY($1)) AND deleted_at IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "a65e5f3ae0b0506fb5204aa83ea49fa70745c460e53ec316be184a309f492754" +} diff --git a/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json b/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json new file mode 100644 index 00000000000..5dd86cd9e50 --- /dev/null +++ b/.sqlx/query-a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO comms_messages (\n id, parent_entity_type, parent_entity_id, thread_id, sender_id, imported_author,\n content, created_at, updated_at, edited_at, deleted_at, import_metadata, import_order\n )\n SELECT cm.message_id, 'document', t.\"documentId\",\n CASE WHEN cm.message_id = tm.root_id THEN NULL ELSE tm.root_id END,\n c.owner, c.sender, c.text,\n c.\"createdAt\" AT TIME ZONE 'UTC', c.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE WHEN c.\"updatedAt\" > c.\"createdAt\" THEN c.\"updatedAt\" END,\n COALESCE(c.\"deletedAt\", t.\"deletedAt\"), c.metadata,\n row_number() OVER (PARTITION BY t.id ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id)\n FROM \"Comment\" c\n JOIN \"Thread\" t ON t.id = c.\"threadId\"\n JOIN migrated_comment_id cm ON cm.comment_id = c.id\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (id) DO UPDATE SET\n content = EXCLUDED.content,\n updated_at = EXCLUDED.updated_at,\n edited_at = EXCLUDED.edited_at,\n deleted_at = EXCLUDED.deleted_at,\n imported_author = EXCLUDED.imported_author,\n import_metadata = EXCLUDED.import_metadata,\n import_order = EXCLUDED.import_order\n WHERE comms_messages.updated_at <= EXCLUDED.updated_at\n AND (comms_messages.content, comms_messages.updated_at, comms_messages.edited_at,\n comms_messages.deleted_at, comms_messages.imported_author,\n comms_messages.import_metadata, comms_messages.import_order)\n IS DISTINCT FROM\n (EXCLUDED.content, EXCLUDED.updated_at, EXCLUDED.edited_at, EXCLUDED.deleted_at,\n EXCLUDED.imported_author, EXCLUDED.import_metadata, EXCLUDED.import_order)\n RETURNING (xmax = 0) AS \"inserted!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inserted!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a6fe5e93cc6b7ed25934216cfa647b8591213be9dc848d874b3b3f2a78226669" +} diff --git a/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json b/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json new file mode 100644 index 00000000000..bae7d688813 --- /dev/null +++ b/.sqlx/query-b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b44f233a1a834c1ae81dbfda9c248a7b7daf2813c73521fbb8691d4f34ca28ea" +} diff --git a/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json b/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json new file mode 100644 index 00000000000..05bb70f5691 --- /dev/null +++ b/.sqlx/query-bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT d.id FROM \"Document\" d\n WHERE ($1::text IS NULL OR d.id > $1)\n AND (EXISTS (SELECT 1 FROM \"Thread\" t WHERE t.\"documentId\" = d.id)\n OR EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.document_id = d.id))\n ORDER BY d.id LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "bc996dcd91b86d30e494387bacf11fc307ac363e4d1701439b0bc6ad59e29d16" +} diff --git a/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json b/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json new file mode 100644 index 00000000000..63e2c7761a3 --- /dev/null +++ b/.sqlx/query-cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT t.id, t.\"documentId\" AS document_id, first.message_id AS \"root_id?\"\n FROM \"Thread\" t\n LEFT JOIN LATERAL (\n SELECT m.message_id FROM \"Comment\" c\n JOIN migrated_comment_id m ON m.comment_id = c.id\n WHERE c.\"threadId\" = t.id\n ORDER BY c.\"order\" NULLS LAST, c.\"createdAt\", c.id LIMIT 1\n ) first ON true\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)\n ORDER BY t.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "document_id", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "root_id?", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "cad533fdf9dff2cb374956bb810b6672179e9e5afc7bc6e059b3ba14f4cd0dd0" +} diff --git a/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json b/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json new file mode 100644 index 00000000000..c831efb9301 --- /dev/null +++ b/.sqlx/query-e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT c.id, t.\"documentId\" AS document_id FROM \"Comment\" c\n JOIN \"Thread\" t ON t.id = c.\"threadId\"\n WHERE t.\"documentId\" = ANY($1)\n AND NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)\n ORDER BY c.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "document_id", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "e558ac1741835623b34dea5518c43f58913909a51b4f5f6ef057f8cccdd8eab7" +} diff --git a/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json b/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json new file mode 100644 index 00000000000..d4071cbc7ae --- /dev/null +++ b/.sqlx/query-e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE comms_messages m SET deleted_at = now(), updated_at = now()\n FROM migrated_comment_id cm\n WHERE cm.message_id = m.id AND cm.document_id = ANY($1) AND m.deleted_at IS NULL\n AND NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.id = cm.comment_id)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e9f33d6edca87d88c7403b13d19fe3aa835c20b17993e9e8cdabfe20ed5efa34" +} diff --git a/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json b/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json new file mode 100644 index 00000000000..e0e7f4d8d06 --- /dev/null +++ b/.sqlx/query-f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(DISTINCT \"documentId\") AS \"count!\" FROM \"Thread\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "f4387247669703803329aa0d090c0b6427a7b17676d1fdf8fdfd950dbf685dd0" +} diff --git a/Cargo.lock b/Cargo.lock index c35d080dfaa..e53cb6ac5e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10134,7 +10134,17 @@ dependencies = [ name = "macro_db_migrator" version = "0.1.0" dependencies = [ + "chrono", + "clap", + "database_env_vars", + "macro_entrypoint", + "macro_env", + "macro_uuid", + "rootcause", + "serde_json", "sqlx", + "tokio", + "tracing", "workspace-hack", ] diff --git a/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql b/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql new file mode 100644 index 00000000000..770566ff01b --- /dev/null +++ b/crates/macro_db_client/migrations/20260917190902_messages_mapping_foreign_keys.sql @@ -0,0 +1,20 @@ +-- The legacy comment importer allocates a mapping row and writes the message +-- it points at inside the same transaction, so these references hold at every +-- commit boundary. Deferred so the mapping can be inserted before the message. +ALTER TABLE migrated_comment_id + ADD CONSTRAINT migrated_comment_id_message_id_fkey + FOREIGN KEY (message_id) REFERENCES comms_messages (id) + ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + +ALTER TABLE migrated_comment_thread_id + ADD CONSTRAINT migrated_comment_thread_id_root_id_fkey + FOREIGN KEY (root_id) REFERENCES comms_message_threads (root_id) + ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + +-- The importer works one batch of documents at a time and the Document FK +-- cascades by document; neither had an index on the referencing column. +CREATE INDEX idx_migrated_comment_id_document + ON migrated_comment_id (document_id); + +CREATE INDEX idx_migrated_comment_thread_id_document + ON migrated_comment_thread_id (document_id); diff --git a/crates/macro_db_migrator/Cargo.toml b/crates/macro_db_migrator/Cargo.toml index 15efd1049b1..6d827198e52 100644 --- a/crates/macro_db_migrator/Cargo.toml +++ b/crates/macro_db_migrator/Cargo.toml @@ -4,6 +4,35 @@ name = "macro_db_migrator" publish = false version = "0.1.0" +[features] +comment-import = [ + "dep:clap", + "dep:database_env_vars", + "dep:macro_entrypoint", + "dep:macro_env", + "dep:macro_uuid", + "dep:rootcause", + "dep:serde_json", + "dep:tokio", + "dep:tracing", +] + +[[bin]] +name = "comment_import" +required-features = ["comment-import"] + [dependencies] +clap = { workspace = true, optional = true } +database_env_vars = { path = "../database_env_vars", optional = true } +macro_entrypoint = { path = "../macro_entrypoint", optional = true } +macro_env = { path = "../macro_env", optional = true } +macro_uuid = { path = "../macro_uuid", optional = true } +rootcause = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } sqlx = { workspace = true, features = ["migrate"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"], optional = true } +tracing = { workspace = true, optional = true } workspace-hack = { version = "0.1", path = "../workspace-hack" } + +[dev-dependencies] +chrono = { workspace = true } diff --git a/crates/macro_db_migrator/src/bin/comment_import.rs b/crates/macro_db_migrator/src/bin/comment_import.rs new file mode 100644 index 00000000000..6496b370b1a --- /dev/null +++ b/crates/macro_db_migrator/src/bin/comment_import.rs @@ -0,0 +1,45 @@ +//! Online, idempotent import of legacy document comments into the shared +//! message store. Safe to rerun while the application keeps serving requests. + +#[path = "comment_import/runner.rs"] +mod runner; + +use clap::Parser; +use database_env_vars::DatabaseUrl; +use macro_entrypoint::MacroEntrypoint; +use macro_env::Environment; +use sqlx::{Connection, PgConnection}; + +#[derive(Debug, Parser)] +#[command( + about = "Import legacy document comments into comms_messages while the application stays live" +)] +struct Args { + /// Documents imported per transaction. Smaller batches hold the legacy + /// table lock for less time; larger batches finish sooner. + #[arg(long, default_value_t = runner::DEFAULT_DOCUMENTS_PER_BATCH)] + documents_per_batch: i64, + /// Run the preflight and report pending work without writing anything. + #[arg(long)] + check: bool, +} + +#[tokio::main] +async fn main() -> Result<(), rootcause::Report> { + let args = Args::parse(); + let entrypoint = MacroEntrypoint::new(Environment::Local).init(); + let database_url = DatabaseUrl::new()?; + let mut connection = PgConnection::connect(database_url.as_ref()).await?; + let report = if args.check { + runner::check(&mut connection).await?.to_string() + } else { + let options = runner::Options { + documents_per_batch: args.documents_per_batch, + }; + runner::run(&mut connection, options).await?.to_string() + }; + println!("{report}"); + connection.close().await?; + entrypoint.shutdown(); + Ok(()) +} diff --git a/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql b/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql new file mode 100644 index 00000000000..d404905602b --- /dev/null +++ b/crates/macro_db_migrator/src/bin/comment_import/legacy_fixture.sql @@ -0,0 +1,46 @@ +INSERT INTO macro_user(id, username, email, stripe_customer_id) +VALUES ('01990000-0000-7000-8000-000000000001', 'migration', 'migration@example.com', 'migration'); +INSERT INTO "User"(id, email, macro_user_id) +VALUES ('macro|migration@example.com', 'migration@example.com', '01990000-0000-7000-8000-000000000001'); +INSERT INTO comms_channels(id, name, channel_type, owner_id) +VALUES ('01990000-0000-7000-8000-000000000005', 'Existing channel', 'private', 'macro|migration@example.com'); +INSERT INTO comms_messages(id, channel_id, thread_id, sender_id, content) VALUES + ('01990000-0000-7000-8000-000000000006', '01990000-0000-7000-8000-000000000005', null, 'macro|migration@example.com', 'Channel root'), + ('01990000-0000-7000-8000-000000000007', '01990000-0000-7000-8000-000000000005', '01990000-0000-7000-8000-000000000006', 'macro|migration@example.com', 'Channel reply'); +INSERT INTO "Document"(id, name, owner, "fileType") VALUES + ('legacy-md', 'Legacy', 'macro|migration@example.com', 'md'), + ('legacy-pdf', 'Legacy PDF', 'macro|migration@example.com', 'pdf'); +INSERT INTO "Thread"(id, owner, "documentId", resolved, metadata, "createdAt", "updatedAt") VALUES + (1, 'macro|migration@example.com', 'legacy-md', true, '{"markId":"01990000-0000-7000-8000-000000000002"}', '2020-01-01', '2020-01-02'), + (2, 'macro|migration@example.com', 'legacy-md', false, '{"markId":"DISCUSSION:legacy"}', '2020-01-05', '2020-01-06'), + (3, 'macro|migration@example.com', 'legacy-pdf', false, '{}', '2020-01-01', '2020-01-01'), + (4, 'macro|migration@example.com', 'legacy-pdf', false, '{}', '2020-01-01', '2020-01-01'), + (5, 'macro|migration@example.com', 'legacy-md', false, '{"markId":"DISCUSSION:01990000-0000-7000-8000-000000000009"}', '2020-01-07', '2020-01-07'); +INSERT INTO "Comment"(id, "threadId", owner, sender, text, "order", "createdAt", "updatedAt", "deletedAt") VALUES + (10, 1, 'macro|migration@example.com', 'Original author', 'Deleted root', 1, '2020-01-01', '2020-01-01', '2020-01-03'), + (11, 1, 'macro|migration@example.com', null, 'Surviving reply', 2, '2020-01-02', '2020-01-02', null), + (12, 3, 'macro|migration@example.com', 'PDF Author', 'PDF original', 1, '2020-01-01', '2020-01-01', null), + (13, 4, 'macro|migration@example.com', null, 'Highlight comment', 1, '2020-01-01', '2020-01-01', null), + (14, 5, 'macro|migration@example.com', null, 'Discussion root', null, '2020-01-07', '2020-01-08', null); +INSERT INTO "PdfPlaceableCommentAnchor"( + uuid, "documentId", owner, page, "wasEdited", "wasDeleted", "shouldLockOnSave", + "originalPage", "originalIndex", "xPct", "yPct", "widthPct", "heightPct", rotation, "threadId" +) VALUES ('01990000-0000-7000-8000-000000000003', 'legacy-pdf', 'macro|migration@example.com', + 2, false, false, false, 2, 0, 0.1, 0.2, 0.3, 0.4, 0, 3); +INSERT INTO "PdfHighlightAnchor"( + uuid, "documentId", owner, page, red, green, blue, alpha, type, text, + "pageViewportWidth", "pageViewportHeight", "threadId" +) VALUES ('01990000-0000-7000-8000-000000000004', 'legacy-pdf', 'macro|migration@example.com', + 1, 255, 255, 0, 0.5, 0, 'Highlighted text', 600, 800, 4); +INSERT INTO "PdfHighlightRect"(top, "left", width, height, "pdfHighlightAnchorId") +VALUES (0.1, 0.2, 0.3, 0.4, '01990000-0000-7000-8000-000000000004'); +INSERT INTO "DocumentInstance"(id, "documentId", sha) VALUES (1, 'legacy-pdf', 'fixture'); +INSERT INTO "DocumentInstanceModificationData"("documentInstanceId", "modificationData") +VALUES (1, '{"pages":[{"highlights":[{"comments":[{"id":12,"text":"PDF original"},{"id":"999","text":"Stale"},{"id":"external-pdf-id","text":"External"}]}]}]}'); +INSERT INTO notification(id, notification_event_type, event_item_id, event_item_type, service_sender, metadata) +SELECT gen_random_uuid(), kind, 'legacy-md', 'document', 'dss', + '{"commentId":11,"threadId":1,"commentText":"Surviving reply"}'::jsonb +FROM unnest(ARRAY['commented_on_document', 'replied_to_document_comment_thread', 'mentioned_in_document_comment']) kind; +INSERT INTO notification(id, notification_event_type, event_item_id, event_item_type, service_sender, metadata) +VALUES (gen_random_uuid(), 'commented_on_document', 'legacy-md', 'document', 'dss', + '{"commentId":424242,"threadId":1,"commentText":"Removed long ago"}'::jsonb); diff --git a/crates/macro_db_migrator/src/bin/comment_import/runner.rs b/crates/macro_db_migrator/src/bin/comment_import/runner.rs new file mode 100644 index 00000000000..790d9d07f82 --- /dev/null +++ b/crates/macro_db_migrator/src/bin/comment_import/runner.rs @@ -0,0 +1,766 @@ +//! Database-only operator tooling. No application request or authorization path. +//! +//! Every run projects the legacy `"Comment"` / `"Thread"` rows onto the shared +//! message tables one batch of documents at a time. Rows whose projection +//! already matches are left alone, so a rerun is a no-op and a run after new +//! legacy writes carries only the delta. + +#[path = "summary.rs"] +mod summary; + +use std::collections::HashMap; +use std::time::Duration; + +use macro_uuid::{Uuid, generate_uuid_v7}; +use sqlx::{Connection, PgConnection, Postgres, Transaction}; +pub(super) use summary::{Pending, Summary}; + +const LOCK_ID: i64 = 732_847_301; +/// Documents handled per transaction when the operator gives no override. +pub(super) const DEFAULT_DOCUMENTS_PER_BATCH: i64 = 250; +const MAPPING_CHUNK: usize = 1_000; +const LOCK_RETRIES: u32 = 5; +const LOCK_RETRY_DELAY: Duration = Duration::from_secs(2); +const LOCK_NOT_AVAILABLE: &str = "55P03"; +const DEADLOCK_DETECTED: &str = "40P01"; +const PREFLIGHT_SAMPLE: i64 = 20; +const UUID_PATTERN: &str = + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"; +const DISCUSSION_MARK_PREFIX: &str = "DISCUSSION:"; +const COMMENT_NOTIFICATION_TYPES: [&str; 3] = [ + "commented_on_document", + "replied_to_document_comment_thread", + "mentioned_in_document_comment", +]; +/// Saved PDF export payloads whose `comments[].id` still carries a legacy id. +const NUMERIC_COMMENT_ID_PATH: &str = + r#"$.**.comments[*].id ? (@.type() == "number" || @ like_regex "^[0-9]+$")"#; + +#[derive(Debug, Clone, Copy)] +pub(super) struct Options { + pub documents_per_batch: i64, +} + +impl Default for Options { + fn default() -> Self { + Self { + documents_per_batch: DEFAULT_DOCUMENTS_PER_BATCH, + } + } +} + +/// The caller owns this dedicated connection; dropping it also releases the lock. +pub(super) async fn run( + connection: &mut PgConnection, + options: Options, +) -> Result { + if options.documents_per_batch < 1 { + rootcause::bail!("documents_per_batch must be at least 1"); + } + with_lock(connection, |connection| { + Box::pin(run_locked(connection, options)) + }) + .await +} + +/// Preflight and pending counts; never writes. +pub(super) async fn check(connection: &mut PgConnection) -> Result { + with_lock(connection, |connection| Box::pin(check_locked(connection))).await +} + +type Locked<'a, T> = + std::pin::Pin> + Send + 'a>>; + +async fn with_lock( + connection: &mut PgConnection, + body: impl for<'a> FnOnce(&'a mut PgConnection) -> Locked<'a, T>, +) -> Result { + let acquired = + sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID) + .fetch_one(&mut *connection) + .await?; + if !acquired { + rootcause::bail!("Another comment import is running"); + } + let result = body(connection).await; + let unlock = sqlx::query_scalar!(r#"SELECT pg_advisory_unlock($1) AS "released!""#, LOCK_ID) + .fetch_one(&mut *connection) + .await; + let outcome = result?; + unlock?; + Ok(outcome) +} + +async fn check_locked(connection: &mut PgConnection) -> Result { + preflight(connection).await?; + let comments_without_mapping = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM "Comment" c + WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id)"# + ) + .fetch_one(&mut *connection) + .await?; + let threads_without_mapping = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM "Thread" t + WHERE NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id)"# + ) + .fetch_one(&mut *connection) + .await?; + let documents_with_legacy_comments = + sqlx::query_scalar!(r#"SELECT count(DISTINCT "documentId") AS "count!" FROM "Thread""#) + .fetch_one(&mut *connection) + .await?; + Ok(Pending { + comments_without_mapping, + threads_without_mapping, + documents_with_legacy_comments, + }) +} + +async fn run_locked( + connection: &mut PgConnection, + options: Options, +) -> Result { + preflight(connection).await?; + let mut summary = Summary::default(); + let mut cursor: Option = None; + loop { + let documents = sqlx::query_scalar!( + r#"SELECT d.id FROM "Document" d + WHERE ($1::text IS NULL OR d.id > $1) + AND (EXISTS (SELECT 1 FROM "Thread" t WHERE t."documentId" = d.id) + OR EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.document_id = d.id)) + ORDER BY d.id LIMIT $2"#, + cursor, + options.documents_per_batch, + ) + .fetch_all(&mut *connection) + .await?; + let Some(last) = documents.last() else { break }; + cursor = Some(last.clone()); + import_batch_with_retry(connection, &documents, &mut summary).await?; + summary.documents += documents.len() as u64; + summary.batches += 1; + } + Ok(summary) +} + +/// Legacy data that cannot be represented and must be resolved by hand first. +async fn preflight(connection: &mut PgConnection) -> Result<(), rootcause::Report> { + let ambiguous = sqlx::query_scalar!( + r#"SELECT t.id FROM "Thread" t + LEFT JOIN "PdfPlaceableCommentAnchor" p ON p."threadId" = t.id + LEFT JOIN "PdfHighlightAnchor" h ON h."threadId" = t.id AND h."deletedAt" IS NULL + WHERE t."deletedAt" IS NULL + GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1 + ORDER BY t.id LIMIT $1"#, + PREFLIGHT_SAMPLE, + ) + .fetch_all(&mut *connection) + .await?; + if !ambiguous.is_empty() { + rootcause::bail!( + "Multiple PDF anchors on a legacy thread; resolve before importing (thread ids {ambiguous:?})" + ); + } + let duplicates = sqlx::query!( + r#"SELECT t."documentId" AS document_id, t.metadata->>'markId' AS "mark_id!", + array_agg(t.id ORDER BY t.id) AS "thread_ids!" + FROM "Thread" t + WHERE t."deletedAt" IS NULL + AND t.metadata->>'markId' ~ $1 + AND EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id) + AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id) + AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h + WHERE h."threadId" = t.id AND h."deletedAt" IS NULL) + GROUP BY 1, 2 HAVING count(*) > 1 + ORDER BY 1, 2 LIMIT $2"#, + UUID_PATTERN, + PREFLIGHT_SAMPLE, + ) + .fetch_all(&mut *connection) + .await?; + if let Some(first) = duplicates.first() { + rootcause::bail!( + "Multiple active legacy threads share the same Markdown mark; delete the duplicates before importing (document {} mark {} threads {:?}, {} conflicts sampled)", + first.document_id, + first.mark_id, + first.thread_ids, + duplicates.len() + ); + } + Ok(()) +} + +async fn import_batch_with_retry( + connection: &mut PgConnection, + documents: &[String], + summary: &mut Summary, +) -> Result<(), rootcause::Report> { + let mut attempt = 0; + loop { + match import_batch(connection, documents, summary).await { + Ok(()) => return Ok(()), + Err(error) if attempt < LOCK_RETRIES && is_retryable(&error) => { + attempt += 1; + tracing::warn!( + error = %error, + attempt, + documents = documents.len(), + "legacy tables busy; retrying batch" + ); + tokio::time::sleep(LOCK_RETRY_DELAY).await; + } + Err(error) => return Err(error.into()), + } + } +} + +fn is_retryable(error: &sqlx::Error) -> bool { + error + .as_database_error() + .and_then(|e| e.code()) + .is_some_and(|code| code == LOCK_NOT_AVAILABLE || code == DEADLOCK_DETECTED) +} + +/// One transaction per batch: mapping allocation and the rows those mappings +/// point at commit together, so the mapping foreign keys hold at every boundary. +async fn import_batch( + connection: &mut PgConnection, + documents: &[String], + summary: &mut Summary, +) -> Result<(), sqlx::Error> { + let mut tx = connection.begin().await?; + lock_legacy_tables(&mut tx).await?; + summary.comment_mappings_allocated += allocate_comment_mappings(&mut tx, documents).await?; + let new_roots = allocate_thread_mappings(&mut tx, documents).await?; + summary.thread_mappings_allocated += new_roots.len() as u64; + + let (inserted, updated) = upsert_messages(&mut tx, documents).await?; + summary.messages_inserted += inserted; + summary.messages_updated += updated; + let (inserted, updated) = upsert_structural_roots(&mut tx, documents).await?; + summary.messages_inserted += inserted; + summary.messages_updated += updated; + summary.messages_tombstoned += tombstone_orphan_messages(&mut tx, documents).await?; + + summary.threads_written += upsert_thread_state(&mut tx, documents, &new_roots).await?; + let (threads, messages) = tombstone_orphan_threads(&mut tx, documents).await?; + summary.threads_tombstoned += threads; + summary.messages_tombstoned += messages; + + summary.anchors_linked += link_anchors(&mut tx, documents).await?; + let (remapped, unmapped) = remap_notifications(&mut tx, documents).await?; + summary.notifications_remapped += remapped; + summary.warnings.unmapped_notifications += unmapped; + let (remapped, unmapped) = remap_pdf_payloads(&mut tx, documents).await?; + summary.pdf_payloads_remapped += remapped; + summary.warnings.unmapped_pdf_comment_ids += unmapped; + + summary.warnings.invalid_mark_ids += count_invalid_mark_ids(&mut tx, documents).await?; + summary.warnings.root_order_drift += count_root_order_drift(&mut tx, documents).await?; + tx.commit().await +} + +#[expect( + clippy::disallowed_methods, + reason = "utility statements carry no result shape for SQLx to check" +)] +async fn lock_legacy_tables(tx: &mut Transaction<'_, Postgres>) -> Result<(), sqlx::Error> { + sqlx::query("SET LOCAL lock_timeout = '5s'") + .execute(&mut **tx) + .await?; + sqlx::query(r#"LOCK TABLE "Comment", "Thread" IN SHARE ROW EXCLUSIVE MODE"#) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn allocate_comment_mappings( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result { + let missing = sqlx::query!( + r#"SELECT c.id, t."documentId" AS document_id FROM "Comment" c + JOIN "Thread" t ON t.id = c."threadId" + WHERE t."documentId" = ANY($1) + AND NOT EXISTS (SELECT 1 FROM migrated_comment_id m WHERE m.comment_id = c.id) + ORDER BY c.id"#, + documents, + ) + .fetch_all(&mut **tx) + .await?; + for chunk in missing.chunks(MAPPING_CHUNK) { + let ids: Vec = chunk.iter().map(|row| row.id).collect(); + let uuids: Vec = chunk.iter().map(|_| generate_uuid_v7()).collect(); + let document_ids: Vec = chunk.iter().map(|row| row.document_id.clone()).collect(); + sqlx::query!( + "INSERT INTO migrated_comment_id (comment_id, message_id, document_id) + SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])", + &ids, + &uuids, + &document_ids, + ) + .execute(&mut **tx) + .await?; + } + Ok(missing.len() as u64) +} + +/// Roots follow legacy display order, then creation time, then id, including +/// deleted comments. Empty threads receive a fresh root id for a structural tombstone. +async fn allocate_thread_mappings( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result, sqlx::Error> { + let missing = sqlx::query!( + r#"SELECT t.id, t."documentId" AS document_id, first.message_id AS "root_id?" + FROM "Thread" t + LEFT JOIN LATERAL ( + SELECT m.message_id FROM "Comment" c + JOIN migrated_comment_id m ON m.comment_id = c.id + WHERE c."threadId" = t.id + ORDER BY c."order" NULLS LAST, c."createdAt", c.id LIMIT 1 + ) first ON true + WHERE t."documentId" = ANY($1) + AND NOT EXISTS (SELECT 1 FROM migrated_comment_thread_id m WHERE m.thread_id = t.id) + ORDER BY t.id"#, + documents, + ) + .fetch_all(&mut **tx) + .await?; + let mut roots = Vec::with_capacity(missing.len()); + for chunk in missing.chunks(MAPPING_CHUNK) { + let ids: Vec = chunk.iter().map(|row| row.id).collect(); + let chunk_roots: Vec = chunk + .iter() + .map(|row| row.root_id.unwrap_or_else(generate_uuid_v7)) + .collect(); + let document_ids: Vec = chunk.iter().map(|row| row.document_id.clone()).collect(); + sqlx::query!( + "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id) + SELECT * FROM UNNEST($1::bigint[], $2::uuid[], $3::text[])", + &ids, + &chunk_roots, + &document_ids, + ) + .execute(&mut **tx) + .await?; + roots.extend(chunk_roots); + } + Ok(roots) +} + +fn count_inserts(rows: &[bool]) -> (u64, u64) { + let inserted = rows.iter().filter(|inserted| **inserted).count() as u64; + (inserted, rows.len() as u64 - inserted) +} + +/// Every mapped comment. Existing rows change only when the legacy row differs +/// and is not older than the stored row, so an edit made through the new API +/// after a legacy row went stale is never reverted. +async fn upsert_messages( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result<(u64, u64), sqlx::Error> { + let rows = sqlx::query_scalar!( + r#"INSERT INTO comms_messages ( + id, parent_entity_type, parent_entity_id, thread_id, sender_id, imported_author, + content, created_at, updated_at, edited_at, deleted_at, import_metadata, import_order + ) + SELECT cm.message_id, 'document', t."documentId", + CASE WHEN cm.message_id = tm.root_id THEN NULL ELSE tm.root_id END, + c.owner, c.sender, c.text, + c."createdAt" AT TIME ZONE 'UTC', c."updatedAt" AT TIME ZONE 'UTC', + CASE WHEN c."updatedAt" > c."createdAt" THEN c."updatedAt" END, + COALESCE(c."deletedAt", t."deletedAt"), c.metadata, + row_number() OVER (PARTITION BY t.id ORDER BY c."order" NULLS LAST, c."createdAt", c.id) + FROM "Comment" c + JOIN "Thread" t ON t.id = c."threadId" + JOIN migrated_comment_id cm ON cm.comment_id = c.id + JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id + WHERE t."documentId" = ANY($1) + ON CONFLICT (id) DO UPDATE SET + content = EXCLUDED.content, + updated_at = EXCLUDED.updated_at, + edited_at = EXCLUDED.edited_at, + deleted_at = EXCLUDED.deleted_at, + imported_author = EXCLUDED.imported_author, + import_metadata = EXCLUDED.import_metadata, + import_order = EXCLUDED.import_order + WHERE comms_messages.updated_at <= EXCLUDED.updated_at + AND (comms_messages.content, comms_messages.updated_at, comms_messages.edited_at, + comms_messages.deleted_at, comms_messages.imported_author, + comms_messages.import_metadata, comms_messages.import_order) + IS DISTINCT FROM + (EXCLUDED.content, EXCLUDED.updated_at, EXCLUDED.edited_at, EXCLUDED.deleted_at, + EXCLUDED.imported_author, EXCLUDED.import_metadata, EXCLUDED.import_order) + RETURNING (xmax = 0) AS "inserted!""#, + documents, + ) + .fetch_all(&mut **tx) + .await?; + Ok(count_inserts(&rows)) +} + +/// Empty legacy threads keep a deleted root so their identity, anchors, and +/// external links stay representable without rendering as a discussion. +async fn upsert_structural_roots( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result<(u64, u64), sqlx::Error> { + let rows = sqlx::query_scalar!( + r#"INSERT INTO comms_messages ( + id, parent_entity_type, parent_entity_id, sender_id, content, + created_at, updated_at, deleted_at + ) + SELECT tm.root_id, 'document', t."documentId", t.owner, '', + t."createdAt" AT TIME ZONE 'UTC', t."updatedAt" AT TIME ZONE 'UTC', + COALESCE(t."deletedAt", t."updatedAt") + FROM "Thread" t + JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id + WHERE t."documentId" = ANY($1) + AND NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id) + ON CONFLICT (id) DO UPDATE SET + updated_at = EXCLUDED.updated_at, + deleted_at = EXCLUDED.deleted_at + WHERE comms_messages.updated_at <= EXCLUDED.updated_at + AND (comms_messages.updated_at, comms_messages.deleted_at) + IS DISTINCT FROM (EXCLUDED.updated_at, EXCLUDED.deleted_at) + RETURNING (xmax = 0) AS "inserted!""#, + documents, + ) + .fetch_all(&mut **tx) + .await?; + Ok(count_inserts(&rows)) +} + +/// A mapped comment whose legacy row was removed outright reads as deleted. +async fn tombstone_orphan_messages( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result { + let result = sqlx::query!( + r#"UPDATE comms_messages m SET deleted_at = now(), updated_at = now() + FROM migrated_comment_id cm + WHERE cm.message_id = m.id AND cm.document_id = ANY($1) AND m.deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c.id = cm.comment_id)"#, + documents, + ) + .execute(&mut **tx) + .await?; + Ok(result.rows_affected()) +} + +/// Thread state for every mapped thread. The root insert trigger has already +/// created a placeholder row for roots written in this batch; those are always +/// overwritten. Older rows follow the same newer-wins rule as messages. +async fn upsert_thread_state( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], + new_roots: &[Uuid], +) -> Result { + let result = sqlx::query!( + r#"INSERT INTO comms_message_threads ( + root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor, + import_metadata, created_at, updated_at, deleted_at + ) + SELECT tm.root_id, 'document', t."documentId", t.owner, t.resolved, + CASE + WHEN pa.uuid IS NOT NULL + THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid) + WHEN ph.uuid IS NOT NULL + THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid) + WHEN t.metadata->>'markId' ~ $3 + THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid) + END, + t.metadata, t."createdAt" AT TIME ZONE 'UTC', t."updatedAt" AT TIME ZONE 'UTC', + CASE + WHEN t."deletedAt" IS NOT NULL THEN t."deletedAt" AT TIME ZONE 'UTC' + WHEN NOT EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id) + THEN t."updatedAt" AT TIME ZONE 'UTC' + END + FROM "Thread" t + JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id + LEFT JOIN LATERAL ( + SELECT a.uuid FROM "PdfPlaceableCommentAnchor" a WHERE a."threadId" = t.id + ORDER BY a.uuid LIMIT 1 + ) pa ON true + LEFT JOIN LATERAL ( + SELECT a.uuid FROM "PdfHighlightAnchor" a + WHERE a."threadId" = t.id AND a."deletedAt" IS NULL + ORDER BY a.uuid LIMIT 1 + ) ph ON true + WHERE t."documentId" = ANY($1) + ON CONFLICT (root_id) DO UPDATE SET + user_id = EXCLUDED.user_id, + resolved = EXCLUDED.resolved, + anchor = EXCLUDED.anchor, + import_metadata = EXCLUDED.import_metadata, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at, + deleted_at = EXCLUDED.deleted_at + WHERE (comms_message_threads.root_id = ANY($2) + OR comms_message_threads.updated_at <= EXCLUDED.updated_at) + AND (comms_message_threads.user_id, comms_message_threads.resolved, + comms_message_threads.anchor, comms_message_threads.import_metadata, + comms_message_threads.created_at, comms_message_threads.updated_at, + comms_message_threads.deleted_at) + IS DISTINCT FROM + (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata, + EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)"#, + documents, + new_roots, + UUID_PATTERN, + ) + .execute(&mut **tx) + .await?; + Ok(result.rows_affected()) +} + +/// A mapped thread whose legacy row was removed outright reads as deleted, with its messages. +async fn tombstone_orphan_threads( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result<(u64, u64), sqlx::Error> { + let roots = sqlx::query_scalar!( + r#"UPDATE comms_message_threads s SET deleted_at = now(), updated_at = now() + FROM migrated_comment_thread_id tm + WHERE tm.root_id = s.root_id AND tm.document_id = ANY($1) AND s.deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM "Thread" t WHERE t.id = tm.thread_id) + RETURNING s.root_id"#, + documents, + ) + .fetch_all(&mut **tx) + .await?; + if roots.is_empty() { + return Ok((0, 0)); + } + let messages = sqlx::query!( + r#"UPDATE comms_messages SET deleted_at = now(), updated_at = now() + WHERE (id = ANY($1) OR thread_id = ANY($1)) AND deleted_at IS NULL"#, + &roots, + ) + .execute(&mut **tx) + .await?; + Ok((roots.len() as u64, messages.rows_affected())) +} + +async fn link_anchors( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result { + let placeables = sqlx::query!( + r#"UPDATE "PdfPlaceableCommentAnchor" a SET root_id = tm.root_id + FROM migrated_comment_thread_id tm + WHERE a."threadId" = tm.thread_id AND a."documentId" = ANY($1) + AND a.root_id IS DISTINCT FROM tm.root_id"#, + documents, + ) + .execute(&mut **tx) + .await?; + let highlights = sqlx::query!( + r#"UPDATE "PdfHighlightAnchor" a SET root_id = tm.root_id + FROM migrated_comment_thread_id tm + WHERE a."threadId" = tm.thread_id AND a."documentId" = ANY($1) + AND a.root_id IS DISTINCT FROM tm.root_id"#, + documents, + ) + .execute(&mut **tx) + .await?; + Ok(placeables.rows_affected() + highlights.rows_affected()) +} + +/// Historical document comment notifications point at message ids once both +/// the comment and its thread have mappings. Rewritten ids never match again. +async fn remap_notifications( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result<(u64, u64), sqlx::Error> { + let types: Vec = COMMENT_NOTIFICATION_TYPES + .iter() + .map(|kind| (*kind).to_owned()) + .collect(); + let remapped = sqlx::query!( + r#"UPDATE notification n SET metadata = jsonb_set( + jsonb_set(n.metadata, '{commentId}', to_jsonb(cm.message_id::text)), + '{threadId}', to_jsonb(tm.root_id::text) + ) + FROM migrated_comment_id cm, migrated_comment_thread_id tm + WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1) + AND n.notification_event_type = ANY($2) + AND cm.document_id = n.event_item_id AND tm.document_id = n.event_item_id + AND n.metadata->>'commentId' = cm.comment_id::text + AND n.metadata->>'threadId' = tm.thread_id::text"#, + documents, + &types, + ) + .execute(&mut **tx) + .await?; + let unmapped = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM notification n + WHERE n.event_item_type = 'document' AND n.event_item_id = ANY($1) + AND n.notification_event_type = ANY($2) + AND (n.metadata->>'commentId' ~ '^[0-9]+$' OR n.metadata->>'threadId' ~ '^[0-9]+$')"#, + documents, + &types, + ) + .fetch_one(&mut **tx) + .await?; + Ok((remapped.rows_affected(), unmapped as u64)) +} + +/// Saved PDF export payloads are relational metadata; their comment ids follow +/// the mapping. Ids without a mapping stay untouched and are counted. +async fn remap_pdf_payloads( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result<(u64, u64), sqlx::Error> { + let payloads = sqlx::query!( + r#"SELECT m.id, i."documentId" AS document_id, m."modificationData" AS payload + FROM "DocumentInstanceModificationData" m + JOIN "DocumentInstance" i ON i.id = m."documentInstanceId" + JOIN "Document" d ON d.id = i."documentId" + WHERE i."documentId" = ANY($1) AND d."fileType" = 'pdf' + AND jsonb_path_exists(m."modificationData", $2::text::jsonpath) + ORDER BY m.id"#, + documents, + NUMERIC_COMMENT_ID_PATH, + ) + .fetch_all(&mut **tx) + .await?; + let mut remapped = 0; + let mut unmapped = 0; + let mut mappings: HashMap> = HashMap::new(); + for row in payloads { + if !mappings.contains_key(&row.document_id) { + let rows = sqlx::query!( + "SELECT comment_id, message_id FROM migrated_comment_id WHERE document_id = $1", + row.document_id, + ) + .fetch_all(&mut **tx) + .await?; + mappings.insert( + row.document_id.clone(), + rows.into_iter() + .map(|mapping| (mapping.comment_id, mapping.message_id)) + .collect(), + ); + } + let mut payload = row.payload; + let stats = remap_comment_ids(&mut payload, &mappings[&row.document_id]); + unmapped += stats.unmapped; + if stats.remapped > 0 { + sqlx::query!( + r#"UPDATE "DocumentInstanceModificationData" SET "modificationData" = $1 WHERE id = $2"#, + payload, + row.id, + ) + .execute(&mut **tx) + .await?; + remapped += 1; + } + } + Ok((remapped, unmapped)) +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct RemapStats { + remapped: u64, + unmapped: u64, +} + +/// Rewrite every `comments[].id` holding a legacy comment id, at any depth. +fn remap_comment_ids(value: &mut serde_json::Value, mappings: &HashMap) -> RemapStats { + let mut stats = RemapStats::default(); + match value { + serde_json::Value::Array(items) => { + for item in items { + let inner = remap_comment_ids(item, mappings); + stats.remapped += inner.remapped; + stats.unmapped += inner.unmapped; + } + } + serde_json::Value::Object(fields) => { + for (key, child) in fields.iter_mut() { + if key == "comments" && child.is_array() { + for comment in child.as_array_mut().into_iter().flatten() { + let Some(id) = comment.get("id").and_then(legacy_comment_id) else { + continue; + }; + match mappings.get(&id) { + Some(message_id) => { + comment["id"] = serde_json::Value::String(message_id.to_string()); + stats.remapped += 1; + } + None => stats.unmapped += 1, + } + } + } else { + let inner = remap_comment_ids(child, mappings); + stats.remapped += inner.remapped; + stats.unmapped += inner.unmapped; + } + } + } + _ => {} + } + stats +} + +fn legacy_comment_id(id: &serde_json::Value) -> Option { + match id { + serde_json::Value::Number(number) => number.as_i64(), + serde_json::Value::String(text) if text.bytes().all(|byte| byte.is_ascii_digit()) => { + text.parse().ok() + } + _ => None, + } +} + +async fn count_invalid_mark_ids( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result { + let count = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM "Thread" t + WHERE t."documentId" = ANY($1) AND t."deletedAt" IS NULL + AND t.metadata->>'markId' IS NOT NULL + AND t.metadata->>'markId' !~ $2 + AND t.metadata->>'markId' NOT LIKE $3 + AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id) + AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h + WHERE h."threadId" = t.id AND h."deletedAt" IS NULL)"#, + documents, + UUID_PATTERN, + format!("{DISCUSSION_MARK_PREFIX}%"), + ) + .fetch_one(&mut **tx) + .await?; + Ok(count as u64) +} + +/// Roots are fixed at first import; a legacy reorder afterwards is reported, not applied. +async fn count_root_order_drift( + tx: &mut Transaction<'_, Postgres>, + documents: &[String], +) -> Result { + let count = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM "Thread" t + JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id + JOIN LATERAL ( + SELECT cm.message_id FROM "Comment" c + JOIN migrated_comment_id cm ON cm.comment_id = c.id + WHERE c."threadId" = t.id + ORDER BY c."order" NULLS LAST, c."createdAt", c.id LIMIT 1 + ) first ON true + WHERE t."documentId" = ANY($1) AND first.message_id <> tm.root_id"#, + documents, + ) + .fetch_one(&mut **tx) + .await?; + Ok(count as u64) +} + +#[cfg(test)] +#[path = "test.rs"] +mod test; diff --git a/crates/macro_db_migrator/src/bin/comment_import/summary.rs b/crates/macro_db_migrator/src/bin/comment_import/summary.rs new file mode 100644 index 00000000000..8b558fe34f7 --- /dev/null +++ b/crates/macro_db_migrator/src/bin/comment_import/summary.rs @@ -0,0 +1,169 @@ +//! Run reports printed to the operator. + +use std::fmt; + +/// What one run wrote. Every counter is zero when the store already matched. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Summary { + pub documents: u64, + pub batches: u64, + pub comment_mappings_allocated: u64, + pub thread_mappings_allocated: u64, + pub messages_inserted: u64, + pub messages_updated: u64, + pub messages_tombstoned: u64, + pub threads_written: u64, + pub threads_tombstoned: u64, + pub anchors_linked: u64, + pub notifications_remapped: u64, + pub pdf_payloads_remapped: u64, + pub warnings: Warnings, +} + +/// Legacy data the import could not represent exactly. Nothing here aborts a run. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Warnings { + /// Document comment notifications still pointing at a legacy id without a mapping. + pub unmapped_notifications: u64, + /// Numeric comment ids in saved PDF payloads without a mapping. + pub unmapped_pdf_comment_ids: u64, + /// Threads whose mark id is neither a UUID nor a `DISCUSSION:` mark; imported unanchored. + pub invalid_mark_ids: u64, + /// Threads whose first legacy comment is no longer the mapped root. + pub root_order_drift: u64, +} + +impl Summary { + /// True when the run changed nothing. + pub fn is_noop(&self) -> bool { + self.comment_mappings_allocated == 0 + && self.thread_mappings_allocated == 0 + && self.messages_inserted == 0 + && self.messages_updated == 0 + && self.messages_tombstoned == 0 + && self.threads_written == 0 + && self.threads_tombstoned == 0 + && self.anchors_linked == 0 + && self.notifications_remapped == 0 + && self.pdf_payloads_remapped == 0 + } +} + +impl fmt::Display for Summary { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_noop() { + writeln!( + f, + "Nothing to import: {} documents with legacy comments already match the message store.", + self.documents + )?; + } else { + writeln!( + f, + "Imported legacy comments across {} documents in {} batches.", + self.documents, self.batches + )?; + writeln!( + f, + " comment mappings allocated: {}", + self.comment_mappings_allocated + )?; + writeln!( + f, + " thread mappings allocated: {}", + self.thread_mappings_allocated + )?; + writeln!( + f, + " messages inserted: {}", + self.messages_inserted + )?; + writeln!(f, " messages updated: {}", self.messages_updated)?; + writeln!( + f, + " messages tombstoned: {}", + self.messages_tombstoned + )?; + writeln!(f, " thread states written: {}", self.threads_written)?; + writeln!( + f, + " thread states tombstoned: {}", + self.threads_tombstoned + )?; + writeln!(f, " PDF anchors linked: {}", self.anchors_linked)?; + writeln!( + f, + " notifications remapped: {}", + self.notifications_remapped + )?; + writeln!( + f, + " PDF payloads remapped: {}", + self.pdf_payloads_remapped + )?; + } + write!(f, "{}", self.warnings) + } +} + +impl fmt::Display for Warnings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let lines = [ + ( + self.unmapped_notifications, + "document comment notifications still reference a legacy id without a mapping", + ), + ( + self.unmapped_pdf_comment_ids, + "numeric comment ids in saved PDF payloads have no mapping and were left as is", + ), + ( + self.invalid_mark_ids, + "threads carry a mark id that is not a UUID and were imported unanchored", + ), + ( + self.root_order_drift, + "threads whose first legacy comment is no longer the mapped root", + ), + ]; + for (count, text) in lines { + if count > 0 { + writeln!(f, " warning: {count} {text}")?; + } + } + Ok(()) + } +} + +/// Work still ahead of the next run, without writing anything. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Pending { + pub comments_without_mapping: i64, + pub threads_without_mapping: i64, + pub documents_with_legacy_comments: i64, +} + +impl fmt::Display for Pending { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Preflight passed.")?; + writeln!( + f, + " documents with legacy comments: {}", + self.documents_with_legacy_comments + )?; + writeln!( + f, + " comments without a mapping: {}", + self.comments_without_mapping + )?; + writeln!( + f, + " threads without a mapping: {}", + self.threads_without_mapping + )?; + write!( + f, + "Edits and deletions of already imported comments are only detected by a real run." + ) + } +} diff --git a/crates/macro_db_migrator/src/bin/comment_import/test.rs b/crates/macro_db_migrator/src/bin/comment_import/test.rs new file mode 100644 index 00000000000..85b1296483f --- /dev/null +++ b/crates/macro_db_migrator/src/bin/comment_import/test.rs @@ -0,0 +1,793 @@ +use super::*; +use macro_db_migrator::MACRO_DB_MIGRATIONS; +use serde_json::{Value, json}; +use sqlx::PgPool; +use summary::Warnings; + +const USER: &str = "macro|migration@example.com"; + +async fn legacy_database(pool: &PgPool) -> PgConnection { + let mut connection = pool.acquire().await.unwrap().detach(); + sqlx::raw_sql(include_str!("legacy_fixture.sql")) + .execute(&mut connection) + .await + .unwrap(); + connection +} + +async fn mappings(connection: &mut PgConnection) -> Vec<(i64, Uuid)> { + sqlx::query!("SELECT comment_id, message_id FROM migrated_comment_id ORDER BY comment_id") + .fetch_all(connection) + .await + .unwrap() + .into_iter() + .map(|row| (row.comment_id, row.message_id)) + .collect() +} + +async fn thread_mappings(connection: &mut PgConnection) -> Vec<(i64, Uuid)> { + sqlx::query!("SELECT thread_id, root_id FROM migrated_comment_thread_id ORDER BY thread_id") + .fetch_all(connection) + .await + .unwrap() + .into_iter() + .map(|row| (row.thread_id, row.root_id)) + .collect() +} + +fn mapped(mappings: &[(i64, Uuid)], legacy_id: i64) -> Uuid { + mappings + .iter() + .find(|(id, _)| *id == legacy_id) + .map(|(_, uuid)| *uuid) + .unwrap() +} + +#[derive(Debug, PartialEq)] +struct StoredMessage { + content: String, + thread_id: Option, + imported_author: Option, + deleted: bool, + import_order: Option, +} + +async fn message(connection: &mut PgConnection, id: Uuid) -> StoredMessage { + sqlx::query!( + r#"SELECT content, thread_id, imported_author, deleted_at IS NOT NULL AS "deleted!", import_order + FROM comms_messages WHERE id = $1"#, + id + ) + .map(|row| StoredMessage { + content: row.content, + thread_id: row.thread_id, + imported_author: row.imported_author, + deleted: row.deleted, + import_order: row.import_order, + }) + .fetch_one(connection) + .await + .unwrap() +} + +#[derive(Debug, PartialEq)] +struct StoredThread { + resolved: bool, + anchor: Option, + deleted: bool, +} + +async fn thread(connection: &mut PgConnection, root_id: Uuid) -> StoredThread { + sqlx::query!( + r#"SELECT resolved, anchor, deleted_at IS NOT NULL AS "deleted!" + FROM comms_message_threads WHERE root_id = $1"#, + root_id + ) + .map(|row| StoredThread { + resolved: row.resolved, + anchor: row.anchor, + deleted: row.deleted, + }) + .fetch_one(connection) + .await + .unwrap() +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn imports_roots_replies_empty_threads_anchors_and_remaps(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + let summary = run(&mut connection, Options::default()).await.unwrap(); + assert_eq!( + summary, + Summary { + documents: 2, + batches: 1, + comment_mappings_allocated: 5, + thread_mappings_allocated: 5, + // Five comments and one structural root for the empty thread. + messages_inserted: 6, + messages_updated: 0, + messages_tombstoned: 0, + threads_written: 5, + threads_tombstoned: 0, + anchors_linked: 2, + notifications_remapped: 3, + pdf_payloads_remapped: 1, + warnings: Warnings { + unmapped_notifications: 1, + unmapped_pdf_comment_ids: 1, + invalid_mark_ids: 0, + root_order_drift: 0, + }, + } + ); + assert!(!summary.is_noop()); + assert!( + summary + .to_string() + .contains("messages inserted: 6") + ); + + let comments = mappings(&mut connection).await; + let threads = thread_mappings(&mut connection).await; + assert_eq!(comments.len(), 5); + assert_eq!(threads.len(), 5); + assert!(comments.iter().all(|(_, id)| id.get_version_num() == 7)); + assert!(threads.iter().all(|(_, id)| id.get_version_num() == 7)); + // Roots follow legacy order: the deleted root keeps its identity as the thread root. + assert_eq!(mapped(&threads, 1), mapped(&comments, 10)); + assert_eq!(mapped(&threads, 3), mapped(&comments, 12)); + assert_eq!(mapped(&threads, 4), mapped(&comments, 13)); + assert_eq!(mapped(&threads, 5), mapped(&comments, 14)); + + let root = message(&mut connection, mapped(&comments, 10)).await; + assert_eq!( + root, + StoredMessage { + content: "Deleted root".into(), + thread_id: None, + imported_author: Some("Original author".into()), + deleted: true, + import_order: Some(1), + } + ); + let reply = message(&mut connection, mapped(&comments, 11)).await; + assert_eq!( + reply, + StoredMessage { + content: "Surviving reply".into(), + thread_id: Some(mapped(&threads, 1)), + imported_author: None, + deleted: false, + import_order: Some(2), + } + ); + let created_at: chrono::DateTime = sqlx::query_scalar!( + r#"SELECT created_at FROM comms_messages WHERE id = $1"#, + mapped(&comments, 11) + ) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(created_at.to_rfc3339(), "2020-01-02T00:00:00+00:00"); + + // Markdown mark, resolution, and an unanchored Discussion thread. + assert_eq!( + thread(&mut connection, mapped(&threads, 1)).await, + StoredThread { + resolved: true, + anchor: Some( + json!({"type": "markdown", "mark_id": "01990000-0000-7000-8000-000000000002"}) + ), + deleted: false, + } + ); + assert_eq!( + thread(&mut connection, mapped(&threads, 5)).await, + StoredThread { + resolved: false, + anchor: None, + deleted: false, + } + ); + // The empty thread is a deleted structural root and a deleted thread. + let structural = message(&mut connection, mapped(&threads, 2)).await; + assert!(structural.deleted && structural.content.is_empty() && structural.thread_id.is_none()); + assert!(comments.iter().all(|(_, id)| *id != mapped(&threads, 2))); + assert!(thread(&mut connection, mapped(&threads, 2)).await.deleted); + + // Notifications point at message ids; the stale one keeps its legacy id. + let notices: Vec = sqlx::query_scalar!( + r#"SELECT metadata AS "metadata!" FROM notification ORDER BY metadata->>'commentText'"# + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(notices.len(), 4); + assert_eq!(notices[0]["commentId"], 424242); + for notice in ¬ices[1..] { + assert_eq!(notice["commentId"], mapped(&comments, 11).to_string()); + assert_eq!(notice["threadId"], mapped(&threads, 1).to_string()); + assert_eq!(notice["commentText"], "Surviving reply"); + } + let pdf: Value = + sqlx::query_scalar!(r#"SELECT "modificationData" FROM "DocumentInstanceModificationData""#) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!( + pdf["pages"][0]["highlights"][0]["comments"], + json!([ + {"id": mapped(&comments, 12), "text": "PDF original"}, + {"id": "999", "text": "Stale"}, + {"id": "external-pdf-id", "text": "External"} + ]) + ); + + // PDF anchors keep their legacy thread id and gain the root; thread state points back. + let placeable = sqlx::query!( + r#"SELECT a."threadId" AS thread_id, a.root_id, a."xPct" AS x_pct, t.anchor + FROM "PdfPlaceableCommentAnchor" a JOIN comms_message_threads t ON t.root_id = a.root_id"# + ) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(placeable.thread_id, Some(3)); + assert_eq!(placeable.root_id, Some(mapped(&threads, 3))); + assert_eq!(placeable.x_pct, 0.1); + assert_eq!( + placeable.anchor, + Some(json!({"type": "pdf_placeable", "anchor_id": "01990000-0000-7000-8000-000000000003"})) + ); + let highlight = sqlx::query!( + r#"SELECT a."threadId" AS thread_id, a.root_id, a.text, r.top, t.anchor + FROM "PdfHighlightAnchor" a JOIN comms_message_threads t ON t.root_id = a.root_id + JOIN "PdfHighlightRect" r ON r."pdfHighlightAnchorId" = a.uuid"# + ) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(highlight.thread_id, Some(4)); + assert_eq!(highlight.root_id, Some(mapped(&threads, 4))); + assert_eq!(highlight.text, "Highlighted text"); + assert_eq!(highlight.top, 0.1); + assert_eq!( + highlight.anchor, + Some(json!({"type": "pdf_highlight", "anchor_id": "01990000-0000-7000-8000-000000000004"})) + ); + + // Legacy rows are untouched and channel messages are not affected. + let legacy_count: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM "Comment""#) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(legacy_count, 5); + let channel_messages: Vec<(String, Option)> = sqlx::query!( + "SELECT content, thread_id FROM comms_messages WHERE parent_entity_type = 'channel' ORDER BY id" + ) + .fetch_all(&mut connection) + .await + .unwrap() + .into_iter() + .map(|row| (row.content, row.thread_id)) + .collect(); + assert_eq!( + channel_messages, + vec![ + ("Channel root".into(), None), + ( + "Channel reply".into(), + Some(Uuid::parse_str("01990000-0000-7000-8000-000000000006").unwrap()) + ), + ] + ); + + // The imported thread accepts new replies and edits through the message store. + let reply_id = generate_uuid_v7(); + sqlx::query!( + "INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, thread_id, sender_id, content) + VALUES ($1, 'document', 'legacy-md', $2, $3, 'After import')", + reply_id, + mapped(&threads, 1), + USER + ) + .execute(&mut connection) + .await + .unwrap(); + + // A second run finds nothing to do and keeps every id. + let again = run(&mut connection, Options::default()).await.unwrap(); + assert!(again.is_noop(), "{again:?}"); + assert_eq!(again.documents, 2); + assert!(again.to_string().starts_with("Nothing to import")); + assert_eq!(again.warnings.unmapped_notifications, 1); + assert_eq!(mappings(&mut connection).await, comments); + assert_eq!(thread_mappings(&mut connection).await, threads); + assert_eq!( + message(&mut connection, reply_id).await.content, + "After import" + ); + + let pending = check(&mut connection).await.unwrap(); + assert_eq!( + pending, + Pending { + comments_without_mapping: 0, + threads_without_mapping: 0, + documents_with_legacy_comments: 2, + } + ); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn delta_runs_carry_new_edited_and_deleted_legacy_comments(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + run(&mut connection, Options::default()).await.unwrap(); + let comments = mappings(&mut connection).await; + let threads = thread_mappings(&mut connection).await; + + // Legacy writers keep going: a new reply, an edit, a comment deletion without + // an updatedAt bump, a thread deletion, and a comment added to the empty thread. + sqlx::raw_sql( + r#" + INSERT INTO "Comment"(id, "threadId", owner, text, "createdAt", "updatedAt") + VALUES (15, 1, 'macro|migration@example.com', 'Late reply', '2020-02-01', '2020-02-01'); + UPDATE "Thread" SET "updatedAt" = '2020-02-01' WHERE id = 1; + UPDATE "Comment" SET text = 'Surviving reply, edited', "updatedAt" = '2020-02-02' WHERE id = 11; + UPDATE "Comment" SET "deletedAt" = '2020-02-03' WHERE id = 14; + UPDATE "Thread" SET "deletedAt" = '2020-02-04' WHERE id = 4; + UPDATE "Comment" SET "deletedAt" = '2020-02-04' WHERE "threadId" = 4; + INSERT INTO "Comment"(id, "threadId", owner, text, "createdAt", "updatedAt") + VALUES (16, 2, 'macro|migration@example.com', 'Revived discussion', '2020-02-05', '2020-02-05'); + UPDATE "Thread" SET "updatedAt" = '2020-02-05' WHERE id = 2; + "#, + ) + .execute(&mut connection) + .await + .unwrap(); + + let delta = run(&mut connection, Options::default()).await.unwrap(); + assert_eq!(delta.comment_mappings_allocated, 2); + assert_eq!(delta.thread_mappings_allocated, 0); + assert_eq!(delta.messages_inserted, 2); + // Edited reply, deleted comment 14, deleted comment 13. The revived thread's + // structural root keeps its tombstone untouched. + assert_eq!(delta.messages_updated, 3); + assert_eq!(delta.messages_tombstoned, 0); + // Thread 1 (updated_at), thread 4 (deleted), thread 2 (revived) changed. + assert_eq!(delta.threads_written, 3); + assert_eq!(delta.warnings.root_order_drift, 1); + + let after = mappings(&mut connection).await; + assert_eq!(after.len(), 7); + assert!(after.starts_with(&comments)); + assert_eq!(thread_mappings(&mut connection).await, threads); + + let late = message(&mut connection, mapped(&after, 15)).await; + assert_eq!(late.content, "Late reply"); + assert_eq!(late.thread_id, Some(mapped(&threads, 1))); + assert_eq!(late.import_order, Some(3)); + let edited = message(&mut connection, mapped(&after, 11)).await; + assert_eq!(edited.content, "Surviving reply, edited"); + assert!(!edited.deleted); + let edited_at: Option = sqlx::query_scalar!( + "SELECT edited_at FROM comms_messages WHERE id = $1", + mapped(&after, 11) + ) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(edited_at.unwrap().to_string(), "2020-02-02 00:00:00"); + assert!(message(&mut connection, mapped(&after, 14)).await.deleted); + assert!(message(&mut connection, mapped(&after, 13)).await.deleted); + assert!(thread(&mut connection, mapped(&threads, 4)).await.deleted); + // The highlight keeps its root so the deleted discussion stays addressable. + let highlight_root: Option = + sqlx::query_scalar!(r#"SELECT root_id FROM "PdfHighlightAnchor""#) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(highlight_root, Some(mapped(&threads, 4))); + + // The formerly empty thread is live again: structural root stays a tombstone, + // the new comment is a reply to it. + let revived = thread(&mut connection, mapped(&threads, 2)).await; + assert!(!revived.deleted); + let revived_comment = message(&mut connection, mapped(&after, 16)).await; + assert_eq!(revived_comment.thread_id, Some(mapped(&threads, 2))); + assert!(message(&mut connection, mapped(&threads, 2)).await.deleted); + + let settled = run(&mut connection, Options::default()).await.unwrap(); + assert!(settled.is_noop(), "{settled:?}"); + assert_eq!(settled.warnings.root_order_drift, 1); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn newer_writes_through_the_message_store_are_never_reverted(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + run(&mut connection, Options::default()).await.unwrap(); + let comments = mappings(&mut connection).await; + let threads = thread_mappings(&mut connection).await; + let reply = mapped(&comments, 11); + + // The new UI edits the imported reply and resolves the Discussion thread + // after the legacy rows went stale. + sqlx::query!( + "UPDATE comms_messages SET content = 'Edited in the new UI', updated_at = now(), edited_at = now() WHERE id = $1", + reply + ) + .execute(&mut connection) + .await + .unwrap(); + sqlx::query!( + "UPDATE comms_message_threads SET resolved = true, updated_at = now() WHERE root_id = $1", + mapped(&threads, 5) + ) + .execute(&mut connection) + .await + .unwrap(); + + let rerun = run(&mut connection, Options::default()).await.unwrap(); + assert!(rerun.is_noop(), "{rerun:?}"); + assert_eq!( + message(&mut connection, reply).await.content, + "Edited in the new UI" + ); + assert!(thread(&mut connection, mapped(&threads, 5)).await.resolved); + + // A legacy edit that is newer than the stored row still wins. + sqlx::query!( + r#"UPDATE "Comment" SET text = 'Edited again in legacy', "updatedAt" = now() + interval '1 hour' WHERE id = 11"# + ) + .execute(&mut connection) + .await + .unwrap(); + let legacy_wins = run(&mut connection, Options::default()).await.unwrap(); + assert_eq!(legacy_wins.messages_updated, 1); + assert_eq!( + message(&mut connection, reply).await.content, + "Edited again in legacy" + ); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn removed_legacy_rows_become_tombstones(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + run(&mut connection, Options::default()).await.unwrap(); + let comments = mappings(&mut connection).await; + let threads = thread_mappings(&mut connection).await; + sqlx::raw_sql( + r#" + DELETE FROM "Comment" WHERE id = 14; + DELETE FROM "Thread" WHERE id = 3; + "#, + ) + .execute(&mut connection) + .await + .unwrap(); + let summary = run(&mut connection, Options::default()).await.unwrap(); + // Comment 14 alone, then thread 3 with its single comment. + assert_eq!(summary.messages_tombstoned, 2); + assert_eq!(summary.threads_tombstoned, 1); + assert!( + message(&mut connection, mapped(&comments, 14)) + .await + .deleted + ); + assert!( + message(&mut connection, mapped(&comments, 12)) + .await + .deleted + ); + assert!(thread(&mut connection, mapped(&threads, 3)).await.deleted); + assert_eq!(mappings(&mut connection).await, comments); + assert!( + run(&mut connection, Options::default()) + .await + .unwrap() + .is_noop() + ); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn rejects_concurrent_runner_ambiguous_anchors_and_duplicate_marks(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + let mut other = pool.acquire().await.unwrap().detach(); + sqlx::query!("SELECT pg_advisory_lock($1)", LOCK_ID) + .execute(&mut other) + .await + .unwrap(); + assert!( + run(&mut connection, Options::default()) + .await + .unwrap_err() + .to_string() + .contains("Another comment import") + ); + other.close().await.unwrap(); + + sqlx::query!(r#"UPDATE "PdfHighlightAnchor" SET "threadId" = 3"#) + .execute(&mut connection) + .await + .unwrap(); + assert!( + run(&mut connection, Options::default()) + .await + .unwrap_err() + .to_string() + .contains("Multiple PDF anchors") + ); + sqlx::query!(r#"UPDATE "PdfHighlightAnchor" SET "threadId" = 4"#) + .execute(&mut connection) + .await + .unwrap(); + + sqlx::query!( + r#"UPDATE "Thread" SET metadata = '{"markId":"01990000-0000-7000-8000-000000000002"}' WHERE id = 5"# + ) + .execute(&mut connection) + .await + .unwrap(); + let error = run(&mut connection, Options::default()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("share the same Markdown mark"), "{error}"); + assert!(error.contains("[1, 5]"), "{error}"); + assert!(mappings(&mut connection).await.is_empty()); + + // The lock is released after a failed preflight. + let acquired: bool = + sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID) + .fetch_one(&mut other_connection(&pool).await) + .await + .unwrap(); + assert!(acquired); + + sqlx::query!(r#"UPDATE "Thread" SET metadata = '{"markId":"not-a-mark"}' WHERE id = 5"#) + .execute(&mut connection) + .await + .unwrap(); + let summary = run(&mut connection, Options::default()).await.unwrap(); + assert_eq!(summary.warnings.invalid_mark_ids, 1); + let threads = thread_mappings(&mut connection).await; + assert_eq!( + thread(&mut connection, mapped(&threads, 5)).await.anchor, + None + ); +} + +async fn other_connection(pool: &PgPool) -> PgConnection { + pool.acquire().await.unwrap().detach() +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn failed_batch_rolls_back_and_can_retry(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + // A discussion created through the message store already owns the mark of + // legacy thread 1, which the preflight cannot see. + let conflicting_root = generate_uuid_v7(); + sqlx::query!( + "INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content) + VALUES ($1, 'document', 'legacy-md', $2, 'New store discussion')", + conflicting_root, + USER + ) + .execute(&mut connection) + .await + .unwrap(); + sqlx::query!( + r#"UPDATE comms_message_threads SET anchor = '{"type":"markdown","mark_id":"01990000-0000-7000-8000-000000000002"}' WHERE root_id = $1"#, + conflicting_root + ) + .execute(&mut connection) + .await + .unwrap(); + + let error = run(&mut connection, Options::default()).await.unwrap_err(); + assert!( + error + .to_string() + .contains("idx_comms_message_threads_markdown_mark"), + "{error}" + ); + assert!(mappings(&mut connection).await.is_empty()); + assert!(thread_mappings(&mut connection).await.is_empty()); + let imported: i64 = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM comms_messages WHERE parent_entity_type = 'document'"# + ) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(imported, 1); + let legacy_count: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM "Comment""#) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(legacy_count, 5); + + sqlx::query!( + "UPDATE comms_message_threads SET anchor = NULL WHERE root_id = $1", + conflicting_root + ) + .execute(&mut connection) + .await + .unwrap(); + let summary = run(&mut connection, Options::default()).await.unwrap(); + assert_eq!(summary.comment_mappings_allocated, 5); + assert_eq!(summary.thread_mappings_allocated, 5); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn mappings_cross_batch_boundaries_and_keep_ids_on_retry(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + let extra = MAPPING_CHUNK as i64 + 100; + sqlx::query!( + r#"INSERT INTO "Thread"(id, owner, "documentId", resolved) + SELECT id, $1, 'legacy-md', false FROM generate_series(100, $2::bigint) id"#, + USER, + extra + ) + .execute(&mut connection) + .await + .unwrap(); + sqlx::query!( + r#"INSERT INTO "Comment"(id, "threadId", owner, text) + SELECT id, id, $1, 'Batch comment' FROM generate_series(100, $2::bigint) id"#, + USER, + extra + ) + .execute(&mut connection) + .await + .unwrap(); + let options = Options { + documents_per_batch: 1, + }; + let summary = run(&mut connection, options).await.unwrap(); + assert_eq!(summary.batches, 2); + assert_eq!(summary.documents, 2); + assert_eq!(summary.comment_mappings_allocated, MAPPING_CHUNK as u64 + 6); + let first = mappings(&mut connection).await; + assert_eq!(first.len(), MAPPING_CHUNK + 6); + assert!(run(&mut connection, options).await.unwrap().is_noop()); + assert_eq!(mappings(&mut connection).await, first); + let count: i64 = sqlx::query_scalar!( + r#"SELECT count(*) AS "count!" FROM comms_message_threads WHERE parent_entity_type = 'document'"# + ) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(count, MAPPING_CHUNK as i64 + 6); +} + +#[test] +fn pdf_comment_ids_are_remapped_at_any_depth_and_unmapped_ids_are_counted() { + let mapped_id = Uuid::from_u128(12); + let mappings = HashMap::from([(12_i64, mapped_id)]); + let mut payload = json!({ + "placeables": [{"payload": {"comments": [{"id": "12", "content": "a"}, {"id": "13", "content": "b"}]}}], + "highlights": {"1": [{"thread": {"comments": [{"id": 12}, {"id": "external"}, "not an object"]}}]}, + "comments": "not an array", + }); + let stats = remap_comment_ids(&mut payload, &mappings); + assert_eq!( + stats, + RemapStats { + remapped: 2, + unmapped: 1, + } + ); + assert_eq!( + payload["placeables"][0]["payload"]["comments"][0]["id"], + mapped_id.to_string() + ); + assert_eq!( + payload["placeables"][0]["payload"]["comments"][1]["id"], + "13" + ); + assert_eq!( + payload["highlights"]["1"][0]["thread"]["comments"][0]["id"], + mapped_id.to_string() + ); + assert_eq!( + payload["highlights"]["1"][0]["thread"]["comments"][1]["id"], + "external" + ); + // A second pass rewrites nothing and still reports the id that has no mapping. + assert_eq!( + remap_comment_ids(&mut payload, &mappings), + RemapStats { + remapped: 0, + unmapped: 1, + } + ); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn channel_read_projections_keep_channel_scoped_index_plans(pool: PgPool) { + let mut connection = legacy_database(&pool).await; + run(&mut connection, Options::default()).await.unwrap(); + // A small target channel among many unrelated channels and non-UUID documents. + // Never disable sequential scans: test the planner's actual chosen access path. + sqlx::raw_sql( + r#" + INSERT INTO comms_channels(id, name, channel_type, owner_id) + SELECT md5('plan-channel-' || n)::uuid, 'Plan fixture', 'private', 'macro|migration@example.com' + FROM generate_series(1, 200) n; + INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content, created_at) + SELECT md5('plan-message-' || c || '-' || n)::uuid, 'channel', md5('plan-channel-' || c)::uuid::text, + 'macro|migration@example.com', 'Unrelated channel', now() - n * interval '1 second' + FROM generate_series(1, 200) c CROSS JOIN generate_series(1, 100) n; + INSERT INTO comms_messages(id, parent_entity_type, parent_entity_id, sender_id, content) + SELECT md5('plan-document-' || n)::uuid, 'document', 'legacy-md', + 'macro|migration@example.com', 'Unrelated document' + FROM generate_series(1, 10000) n; + ANALYZE comms_messages; + "#, + ) + .execute(&mut connection) + .await + .unwrap(); + let channel = Uuid::parse_str("01990000-0000-7000-8000-000000000005").unwrap(); + let queries = [ + // Timeline includes tombstone roots with surviving replies, in either direction. + "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text AND m.thread_id IS NULL + AND (m.deleted_at IS NULL OR EXISTS (SELECT 1 FROM comms_messages r WHERE r.thread_id = m.id AND r.deleted_at IS NULL)) + ORDER BY m.created_at DESC, m.id DESC LIMIT 50", + "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text AND m.thread_id IS NULL + AND (m.deleted_at IS NULL OR EXISTS (SELECT 1 FROM comms_messages r WHERE r.thread_id = m.id AND r.deleted_at IS NULL)) + ORDER BY m.created_at ASC, m.id ASC LIMIT 50", + // Batched latest reads use a parameterized lateral scan for each channel. + "SELECT latest.id FROM unnest(ARRAY[$1::uuid]) i(channel_id) LEFT JOIN LATERAL ( + SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = i.channel_id::text AND m.deleted_at IS NULL + ORDER BY m.created_at DESC LIMIT 1) latest ON true", + "SELECT latest.id FROM unnest(ARRAY[$1::uuid]) i(channel_id) LEFT JOIN LATERAL ( + SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = i.channel_id::text AND m.deleted_at IS NULL AND m.thread_id IS NULL + ORDER BY m.created_at DESC LIMIT 1) latest ON true", + // Unfiltered reads still need a channel index, including deleted replies. + "SELECT m.id FROM comms_messages m WHERE m.parent_entity_type = 'channel' AND m.parent_entity_id = $1::uuid::text ORDER BY m.created_at DESC LIMIT 50", + ]; + for query in queries { + let plan = explain(&mut connection, query, channel).await; + let mut nodes = vec![]; + collect_plan_nodes(&plan[0]["Plan"], &mut nodes); + assert!( + nodes.iter().any(|node| { + node["Index Name"] + .as_str() + .is_some_and(|name| name.starts_with("idx_comms_messages_parent_")) + && node["Index Cond"] + .as_str() + .is_some_and(|condition| condition.contains("parent_entity_id")) + }), + "Channel predicate must be an index condition, not a post-scan filter: {plan:#}" + ); + assert!( + nodes + .iter() + .all(|node| node["Rows Removed by Filter"].as_u64().unwrap_or(0) < 100), + "Channel reads must not filter thousands of unrelated messages: {plan:#}" + ); + } +} + +#[expect( + clippy::disallowed_methods, + reason = "EXPLAIN over test-provided SQL cannot be checked at compile time" +)] +async fn explain(connection: &mut PgConnection, query: &str, channel: Uuid) -> Value { + sqlx::query_scalar(&format!("EXPLAIN (ANALYZE, FORMAT JSON) {query}")) + .bind(channel) + .fetch_one(connection) + .await + .unwrap() +} + +fn collect_plan_nodes<'a>(node: &'a Value, nodes: &mut Vec<&'a Value>) { + nodes.push(node); + if let Some(children) = node["Plans"].as_array() { + for child in children { + collect_plan_nodes(child, nodes); + } + } +} diff --git a/crates/messages/src/outbound/pg_message_repo/test.rs b/crates/messages/src/outbound/pg_message_repo/test.rs index caeb475b4e5..6109c5864ed 100644 --- a/crates/messages/src/outbound/pg_message_repo/test.rs +++ b/crates/messages/src/outbound/pg_message_repo/test.rs @@ -1082,3 +1082,69 @@ async fn both_parents_use_bounded_previews_and_bidirectional_windows(pool: PgPoo ); } } + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn legacy_ids_resolve_through_the_import_mapping_tables(pool: PgPool) { + setup(&pool).await; + let repo = PgMessageRepository::new(pool.clone()); + let parent = MessageParent::parse("document", "message-doc-a").unwrap(); + let root = repo + .create(command("message-doc-a", None, "imported root")) + .await + .unwrap(); + let reply = repo + .create(command("message-doc-a", Some(root.id), "imported reply")) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO migrated_comment_thread_id (thread_id, root_id, document_id) VALUES (1, $1, 'message-doc-a')", + root.id + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO migrated_comment_id (comment_id, message_id, document_id) + VALUES (10, $1, 'message-doc-a'), (11, $2, 'message-doc-a')", + root.id, + reply.id + ) + .execute(&pool) + .await + .unwrap(); + + assert_eq!( + repo.resolve_legacy(&parent, 11, false).await.unwrap(), + Some(reply.id) + ); + assert_eq!( + repo.resolve_legacy(&parent, 10, false).await.unwrap(), + Some(root.id) + ); + assert_eq!( + repo.resolve_legacy(&parent, 1, true).await.unwrap(), + Some(root.id) + ); + // A comment id is not a thread id, and a mapping only resolves under its own document. + assert_eq!(repo.resolve_legacy(&parent, 11, true).await.unwrap(), None); + let other = MessageParent::parse("document", "message-doc-b").unwrap(); + assert_eq!(repo.resolve_legacy(&other, 11, false).await.unwrap(), None); + let channel = MessageParent::parse("channel", &Uuid::from_u128(7).to_string()).unwrap(); + assert_eq!( + repo.resolve_legacy(&channel, 11, false).await.unwrap(), + None + ); + + // Mappings cannot point at messages that were never written. + let dangling = sqlx::query!( + "INSERT INTO migrated_comment_id (comment_id, message_id, document_id) VALUES (12, $1, 'message-doc-a')", + macro_uuid::generate_uuid_v7() + ) + .execute(&pool) + .await + .unwrap_err(); + assert_eq!( + dangling.as_database_error().unwrap().code().as_deref(), + Some("23503") + ); +} diff --git a/docs/legacy-comment-import.md b/docs/legacy-comment-import.md new file mode 100644 index 00000000000..bfebdda6e91 --- /dev/null +++ b/docs/legacy-comment-import.md @@ -0,0 +1,138 @@ +# Legacy comment import + +Moves the historical document comments in `"Comment"` / `"Thread"` into the +shared message store (`comms_messages`, `comms_message_threads`) while the +application keeps running. Nothing is dropped; the legacy tables and endpoints +go away in the last PR of the channel and comment unification stack. + +## Operating model + +The importer is a projection: each run recomputes what every legacy thread and +comment should look like in the message store and writes only the rows whose +projection differs. That gives three properties operators rely on: + +- **Re-runnable.** A second run without legacy changes prints + `Nothing to import` and writes nothing. Ids never change between runs: every + comment and thread gets one UUIDv7 in `migrated_comment_id` / + `migrated_comment_thread_id` on its first sighting and keeps it. +- **Delta-aware.** Comments created, edited, or deleted (soft or hard) through + the legacy endpoints after a run are picked up by the next run. New comments + get mappings; changed ones are compared by text, timestamps, and deletion. +- **Live.** Work happens in one transaction per batch of documents (250 by + default). Each batch takes a `SHARE ROW EXCLUSIVE` lock on `"Comment"` and + `"Thread"` for the duration of that transaction, so legacy comment writes + wait for at most one batch; they are never rejected. Reads are unaffected. + The lock waits at most five seconds for in-flight writers and the batch is + retried. Nothing takes `ACCESS EXCLUSIVE`. + +Two runs cannot overlap: a session advisory lock makes the second exit with +`Another comment import is running`. + +### Sequence + +1. Run the importer as often as you like while legacy writes continue. + Each run reports what it wrote and what it could not represent. +2. Before enabling the new document discussion UI, freeze legacy comment + writes: set `LEGACY_COMMENT_WRITES_ENABLED=false` on the document storage + service (Doppler project `cloud-storage-service`) and roll it out. Comment + create, edit, and delete and anchor delete then answer `503` with + `Document comments are read-only while they move to the new message store`. + Reads, highlight creation, and anchor edits keep working. +3. Run the importer once more. It must end with `Nothing to import` on a + second invocation. +4. Enable the new UI. Do not run the importer again after that unless legacy + writes are frozen; an edit made through the new API is never reverted by a + stale legacy row (newer `updated_at` wins), but a legacy write and a new + store write on the same thread while both are live is a race. + +## Running it + +Build from the repository root; the binary is feature gated so it never ships +with a service: + +```sh +cargo build -p macro_db_migrator --features comment-import --bin comment_import +``` + +Point `DATABASE_URL` at the target. For RDS use the existing helper and never +the repository's local database recipes: + +```sh +export DATABASE_URL="$(tooling/scripts/rds_database_url.sh )" +cargo run -p macro_db_migrator --features comment-import --bin comment_import -- --check +cargo run -p macro_db_migrator --features comment-import --bin comment_import +cargo run -p macro_db_migrator --features comment-import --bin comment_import -- --documents-per-batch 50 +``` + +`--check` runs the preflight and prints how many comments and threads still +lack a mapping without writing. Edits and deletions of already imported +comments are only detected by a real run. + +## What a run writes + +- `migrated_comment_id` / `migrated_comment_thread_id`: one row per legacy + comment and thread. Roots follow legacy display `order`, then `createdAt`, + then id, including deleted roots. An empty thread gets a fresh root id. +- `comms_messages`: one row per comment with `parent_entity_type = 'document'`, + the thread's root as `thread_id`, `sender_id` from `owner`, `imported_author` + from `sender`, `import_metadata` from the comment metadata, and + `import_order` from the legacy order. A deleted root stays a tombstone with + its replies intact. An empty thread gets an empty, deleted structural root. +- `comms_message_threads`: resolution, legacy metadata, and the anchor: + `pdf_placeable` when a placeable anchor points at the thread, else + `pdf_highlight` for a live highlight, else `markdown` when the thread's + `markId` is a UUID. `DISCUSSION:` marks, missing marks, and marks that are + not UUIDs land unanchored. Threads with no comments are marked deleted, so + the new UI never shows an empty "This message was deleted." discussion. +- `"PdfPlaceableCommentAnchor".root_id` / `"PdfHighlightAnchor".root_id` from + the thread mapping; the legacy `"threadId"` stays until the contract PR. +- `notification.metadata` `commentId` / `threadId` for + `commented_on_document`, `replied_to_document_comment_thread`, and + `mentioned_in_document_comment` rows on the batch's documents. +- Numeric `comments[].id` values inside saved PDF + `"DocumentInstanceModificationData"` payloads. + +Rows written through the message store after import are protected: a legacy +row only overwrites a message or thread state when it is at least as new as the +stored `updated_at`. Legacy deletions do not bump `updatedAt`, so they still +apply. + +## Preflight failures + +The run stops before writing when the legacy data cannot be represented: + +- A live thread with more than one PDF anchor. Detach or delete one. +- Two live legacy threads with comments sharing the same Markdown mark on one + document. Delete the duplicate; the message store allows one active + discussion per mark. + +A batch that fails for another reason (for example a discussion created through +the new API already owns a legacy thread's mark) rolls back that batch alone. +Earlier batches stay committed and the failed one is retried on the next run. + +## Warnings + +Warnings are reported and never abort a run: + +- notifications whose `commentId` or `threadId` has no mapping (the comment was + removed before the import existed); they keep their legacy ids. +- numeric PDF comment ids without a mapping; they are left as they are. +- threads whose mark is neither a UUID nor a `DISCUSSION:` mark; imported + unanchored. +- threads whose first legacy comment is no longer the mapped root; roots are + fixed at first import, later comments are replies. + +## Tests + +From the repository root with `SQLX_OFFLINE` unset and local Postgres running: + +```sh +cargo test -p macro_db_migrator --features comment-import +``` + +The suite seeds the legacy tables on the current schema and covers roots, +replies, deleted roots, empty threads, Markdown and PDF anchors, notification +and PDF payload remaps, the no-op second run, delta runs, protection of newer +message store writes, tombstones for removed legacy rows, concurrent runners, +preflight aborts, rollback of a failed batch, mapping batches beyond 1,000 rows, +and the parent-scoped index plans for channel reads. diff --git a/services/document_storage_service/src/api/annotations.rs b/services/document_storage_service/src/api/annotations.rs index e602d571b44..6fda73daffc 100644 --- a/services/document_storage_service/src/api/annotations.rs +++ b/services/document_storage_service/src/api/annotations.rs @@ -5,6 +5,7 @@ pub mod delete_comment; pub mod edit_anchor; pub mod edit_comment; pub mod get; +pub mod write_switch; use std::collections::HashSet; @@ -27,6 +28,12 @@ use notification::domain::models::SendNotificationRequestBuilder; use tower::ServiceBuilder; pub fn router(state: ApiContext) -> Router { + let write_switch = axum::middleware::from_fn_with_state( + write_switch::LegacyCommentWrites { + enabled: state.config.legacy_comment_writes_enabled, + }, + write_switch::handler, + ); Router::new() .route( "/comments/document/{document_id}", @@ -39,22 +46,27 @@ pub fn router(state: ApiContext) -> Router { ) .route( "/comments/document/{document_id}", - post(create_comment::create_comment_handler).layer(ServiceBuilder::new().layer( - axum::middleware::from_fn_with_state( - state.clone(), - macro_middleware::cloud_storage::document::ensure_document_exists::handler, + post(create_comment::create_comment_handler).layer( + ServiceBuilder::new().layer(write_switch.clone()).layer( + axum::middleware::from_fn_with_state( + state.clone(), + macro_middleware::cloud_storage::document::ensure_document_exists::handler, + ), ), - )), + ), ) .route( "/comments/comment/{comment_id}", - delete(delete_comment::delete_comment_handler), + delete(delete_comment::delete_comment_handler).layer(write_switch.clone()), + ) + .route( + "/anchors", + delete(delete_anchor::delete_anchor_handler).layer(write_switch.clone()), ) - .route("/anchors", delete(delete_anchor::delete_anchor_handler)) .route("/anchors", patch(edit_anchor::edit_anchor_handler)) .route( "/comments/comment/{comment_id}", - patch(edit_comment::edit_comment_handler), + patch(edit_comment::edit_comment_handler).layer(write_switch), ) .route( "/anchors/document/{document_id}", diff --git a/services/document_storage_service/src/api/annotations/write_switch.rs b/services/document_storage_service/src/api/annotations/write_switch.rs new file mode 100644 index 00000000000..fe3e3fc89f4 --- /dev/null +++ b/services/document_storage_service/src/api/annotations/write_switch.rs @@ -0,0 +1,39 @@ +use axum::{ + Json, + extract::{Request, State}, + http::StatusCode, + middleware::Next, + response::{IntoResponse, Response}, +}; +use model::response::ErrorResponse; + +/// Whether the legacy comment tables still accept writes through this service. +#[derive(Debug, Clone, Copy)] +pub struct LegacyCommentWrites { + pub enabled: bool, +} + +/// Shown to clients that still write through the legacy comment endpoints +/// while the final import into the shared message store runs. +pub const DISABLED_MESSAGE: &str = "Document comments are read-only while they move to the new message store. Refresh the app in a few minutes and try again."; + +/// Answers 503 for legacy comment writes once the operator has frozen them. +pub async fn handler( + State(switch): State, + request: Request, + next: Next, +) -> Response { + if switch.enabled { + return next.run(request).await; + } + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(ErrorResponse { + message: DISABLED_MESSAGE.into(), + }), + ) + .into_response() +} + +#[cfg(test)] +mod test; diff --git a/services/document_storage_service/src/api/annotations/write_switch/test.rs b/services/document_storage_service/src/api/annotations/write_switch/test.rs new file mode 100644 index 00000000000..79beece009d --- /dev/null +++ b/services/document_storage_service/src/api/annotations/write_switch/test.rs @@ -0,0 +1,37 @@ +use super::*; +use axum::{Router, body::Body, http::Request, routing::post}; +use tower::ServiceExt as _; + +fn router(enabled: bool) -> Router { + Router::new() + .route("/comments", post(|| async { StatusCode::OK })) + .layer(axum::middleware::from_fn_with_state( + LegacyCommentWrites { enabled }, + handler, + )) +} + +fn write_request() -> Request { + Request::builder() + .method("POST") + .uri("/comments") + .body(Body::empty()) + .unwrap() +} + +#[tokio::test] +async fn enabled_switch_passes_writes_through() { + let response = router(true).oneshot(write_request()).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn disabled_switch_answers_service_unavailable_with_the_reason() { + let response = router(false).oneshot(write_request()).await.unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let error: ErrorResponse = serde_json::from_slice(&body).unwrap(); + assert_eq!(error.message, DISABLED_MESSAGE); +} diff --git a/services/document_storage_service/src/config.rs b/services/document_storage_service/src/config.rs index 5125e641024..b4e47687456 100644 --- a/services/document_storage_service/src/config.rs +++ b/services/document_storage_service/src/config.rs @@ -157,6 +157,14 @@ pub struct Config { #[macro_config_default(false)] pub calendar_reminder_dispatch_enabled: bool, + /// Master switch for the legacy document comment writers: comment create, + /// edit and delete, and anchor delete, which also deletes the thread. + /// Set to `false` for the final pass of the legacy comment importer, before + /// the new document discussion UI is enabled; those handlers then answer + /// 503 and the importer works from a frozen source. + #[macro_config_default(true)] + pub legacy_comment_writes_enabled: bool, + /// The number of seconds a signed document or call recording URL is valid for. #[macro_config_default(DEFAULT_PRESIGNED_URL_EXPIRY_SECONDS)] pub document_storage_service_presigned_url_expiry_seconds: u64, From b9248b0cdb0e0a930eaaa0b7497239c693a73cc0 Mon Sep 17 00:00:00 2001 From: Gabriel Birman <25272206+gbirman@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:24:55 +0000 Subject: [PATCH 2/3] fix(messages): count an import batch only after it commits --- .../src/bin/comment_import/runner.rs | 41 +++++++++++-------- .../src/bin/comment_import/summary.rs | 20 +++++++++ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/crates/macro_db_migrator/src/bin/comment_import/runner.rs b/crates/macro_db_migrator/src/bin/comment_import/runner.rs index 790d9d07f82..a3ebd38590c 100644 --- a/crates/macro_db_migrator/src/bin/comment_import/runner.rs +++ b/crates/macro_db_migrator/src/bin/comment_import/runner.rs @@ -224,41 +224,46 @@ fn is_retryable(error: &sqlx::Error) -> bool { /// One transaction per batch: mapping allocation and the rows those mappings /// point at commit together, so the mapping foreign keys hold at every boundary. +/// Counts join the run summary only once the batch has committed, so a retried +/// batch is counted once. async fn import_batch( connection: &mut PgConnection, documents: &[String], summary: &mut Summary, ) -> Result<(), sqlx::Error> { let mut tx = connection.begin().await?; + let mut batch = Summary::default(); lock_legacy_tables(&mut tx).await?; - summary.comment_mappings_allocated += allocate_comment_mappings(&mut tx, documents).await?; + batch.comment_mappings_allocated = allocate_comment_mappings(&mut tx, documents).await?; let new_roots = allocate_thread_mappings(&mut tx, documents).await?; - summary.thread_mappings_allocated += new_roots.len() as u64; + batch.thread_mappings_allocated = new_roots.len() as u64; let (inserted, updated) = upsert_messages(&mut tx, documents).await?; - summary.messages_inserted += inserted; - summary.messages_updated += updated; + batch.messages_inserted += inserted; + batch.messages_updated += updated; let (inserted, updated) = upsert_structural_roots(&mut tx, documents).await?; - summary.messages_inserted += inserted; - summary.messages_updated += updated; - summary.messages_tombstoned += tombstone_orphan_messages(&mut tx, documents).await?; + batch.messages_inserted += inserted; + batch.messages_updated += updated; + batch.messages_tombstoned += tombstone_orphan_messages(&mut tx, documents).await?; - summary.threads_written += upsert_thread_state(&mut tx, documents, &new_roots).await?; + batch.threads_written = upsert_thread_state(&mut tx, documents, &new_roots).await?; let (threads, messages) = tombstone_orphan_threads(&mut tx, documents).await?; - summary.threads_tombstoned += threads; - summary.messages_tombstoned += messages; + batch.threads_tombstoned = threads; + batch.messages_tombstoned += messages; - summary.anchors_linked += link_anchors(&mut tx, documents).await?; + batch.anchors_linked = link_anchors(&mut tx, documents).await?; let (remapped, unmapped) = remap_notifications(&mut tx, documents).await?; - summary.notifications_remapped += remapped; - summary.warnings.unmapped_notifications += unmapped; + batch.notifications_remapped = remapped; + batch.warnings.unmapped_notifications = unmapped; let (remapped, unmapped) = remap_pdf_payloads(&mut tx, documents).await?; - summary.pdf_payloads_remapped += remapped; - summary.warnings.unmapped_pdf_comment_ids += unmapped; + batch.pdf_payloads_remapped = remapped; + batch.warnings.unmapped_pdf_comment_ids = unmapped; - summary.warnings.invalid_mark_ids += count_invalid_mark_ids(&mut tx, documents).await?; - summary.warnings.root_order_drift += count_root_order_drift(&mut tx, documents).await?; - tx.commit().await + batch.warnings.invalid_mark_ids = count_invalid_mark_ids(&mut tx, documents).await?; + batch.warnings.root_order_drift = count_root_order_drift(&mut tx, documents).await?; + tx.commit().await?; + summary.absorb(batch); + Ok(()) } #[expect( diff --git a/crates/macro_db_migrator/src/bin/comment_import/summary.rs b/crates/macro_db_migrator/src/bin/comment_import/summary.rs index 8b558fe34f7..4e1bedd7b82 100644 --- a/crates/macro_db_migrator/src/bin/comment_import/summary.rs +++ b/crates/macro_db_migrator/src/bin/comment_import/summary.rs @@ -34,6 +34,26 @@ pub struct Warnings { } impl Summary { + /// Add a committed batch to the run. + pub fn absorb(&mut self, batch: Self) { + self.documents += batch.documents; + self.batches += batch.batches; + self.comment_mappings_allocated += batch.comment_mappings_allocated; + self.thread_mappings_allocated += batch.thread_mappings_allocated; + self.messages_inserted += batch.messages_inserted; + self.messages_updated += batch.messages_updated; + self.messages_tombstoned += batch.messages_tombstoned; + self.threads_written += batch.threads_written; + self.threads_tombstoned += batch.threads_tombstoned; + self.anchors_linked += batch.anchors_linked; + self.notifications_remapped += batch.notifications_remapped; + self.pdf_payloads_remapped += batch.pdf_payloads_remapped; + self.warnings.unmapped_notifications += batch.warnings.unmapped_notifications; + self.warnings.unmapped_pdf_comment_ids += batch.warnings.unmapped_pdf_comment_ids; + self.warnings.invalid_mark_ids += batch.warnings.invalid_mark_ids; + self.warnings.root_order_drift += batch.warnings.root_order_drift; + } + /// True when the run changed nothing. pub fn is_noop(&self) -> bool { self.comment_mappings_allocated == 0 From e57f0139b83a6673bcbd9c2e58d1878da302bab1 Mon Sep 17 00:00:00 2001 From: Gabriel Birman <25272206+gbirman@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:25:47 +0000 Subject: [PATCH 3/3] fix(messages): address review findings in the comment importer - Exclude wasDeleted PDF placeables from anchor selection and the multi-anchor preflight, matching the deletedAt filter on highlights, so a deleted pin cannot outrank a live highlight or abort the import. - Clear lock_timeout after LOCK TABLE so later batch statements wait normally for unrelated row locks instead of aborting after five seconds. - Recurse into each PDF comment object so ids in nested comments arrays are remapped and counted. - Close the advisory-lock probe connection in the concurrency test. - Match the documented 503 text to DISABLED_MESSAGE. - Refresh the workspace dependency closures against main. --- .github/workspace-dep-closures.json | 5 +++ ...8fa9ab8a3df340502c7e416aa8d445e5f5b86.json | 16 --------- ...8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json | 22 ++++++++++++ ...f2a9dde0e7f00dd58622870b2e7b3e49b79f2.json | 22 ------------ ...62648d83a046b36d7d83ed963c3842657a41.json} | 4 +-- ...b2a37d52cfbf3cf650d1965df042056ad9ce.json} | 4 +-- ...a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json | 16 +++++++++ .../src/bin/comment_import/runner.rs | 34 ++++++++++++------- .../src/bin/comment_import/test.rs | 17 ++++++++-- docs/legacy-comment-import.md | 2 +- 10 files changed, 84 insertions(+), 58 deletions(-) delete mode 100644 .sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json create mode 100644 .sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json delete mode 100644 .sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json rename .sqlx/{query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json => query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json} (72%) rename .sqlx/{query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json => query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json} (72%) create mode 100644 .sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json index 431d81b864d..25e5a992d93 100644 --- a/.github/workspace-dep-closures.json +++ b/.github/workspace-dep-closures.json @@ -5690,7 +5690,12 @@ "crates/workspace-hack" ], "macro_db_migrator": [ + "crates/database_env_vars", "crates/macro_db_migrator", + "crates/macro_entrypoint", + "crates/macro_env", + "crates/macro_env_var", + "crates/macro_uuid", "crates/workspace-hack" ], "macro_entrypoint": [ diff --git a/.sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json b/.sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json deleted file mode 100644 index f710cda86bb..00000000000 --- a/.sqlx/query-3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO comms_message_threads (\n root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,\n import_metadata, created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, t.resolved,\n CASE\n WHEN pa.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)\n WHEN ph.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)\n WHEN t.metadata->>'markId' ~ $3\n THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)\n END,\n t.metadata, t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE\n WHEN t.\"deletedAt\" IS NOT NULL THEN t.\"deletedAt\" AT TIME ZONE 'UTC'\n WHEN NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n THEN t.\"updatedAt\" AT TIME ZONE 'UTC'\n END\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfPlaceableCommentAnchor\" a WHERE a.\"threadId\" = t.id\n ORDER BY a.uuid LIMIT 1\n ) pa ON true\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfHighlightAnchor\" a\n WHERE a.\"threadId\" = t.id AND a.\"deletedAt\" IS NULL\n ORDER BY a.uuid LIMIT 1\n ) ph ON true\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (root_id) DO UPDATE SET\n user_id = EXCLUDED.user_id,\n resolved = EXCLUDED.resolved,\n anchor = EXCLUDED.anchor,\n import_metadata = EXCLUDED.import_metadata,\n created_at = EXCLUDED.created_at,\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE (comms_message_threads.root_id = ANY($2)\n OR comms_message_threads.updated_at <= EXCLUDED.updated_at)\n AND (comms_message_threads.user_id, comms_message_threads.resolved,\n comms_message_threads.anchor, comms_message_threads.import_metadata,\n comms_message_threads.created_at, comms_message_threads.updated_at,\n comms_message_threads.deleted_at)\n IS DISTINCT FROM\n (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,\n EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "TextArray", - "UuidArray", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3e2ff3175e036875d833deaa6868fa9ab8a3df340502c7e416aa8d445e5f5b86" -} diff --git a/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json b/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json new file mode 100644 index 00000000000..f94c37e7261 --- /dev/null +++ b/.sqlx/query-48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT t.id FROM \"Thread\" t\n LEFT JOIN \"PdfPlaceableCommentAnchor\" p ON p.\"threadId\" = t.id AND NOT p.\"wasDeleted\"\n LEFT JOIN \"PdfHighlightAnchor\" h ON h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL\n WHERE t.\"deletedAt\" IS NULL\n GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1\n ORDER BY t.id LIMIT $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "48fbddcebd915244c16d53a6f2c8b68ad4ffd6735cd44817ee2cec34aa6bbe9c" +} diff --git a/.sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json b/.sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json deleted file mode 100644 index 3fda437afbe..00000000000 --- a/.sqlx/query-6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT t.id FROM \"Thread\" t\n LEFT JOIN \"PdfPlaceableCommentAnchor\" p ON p.\"threadId\" = t.id\n LEFT JOIN \"PdfHighlightAnchor\" h ON h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL\n WHERE t.\"deletedAt\" IS NULL\n GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1\n ORDER BY t.id LIMIT $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "6bfddfadc9d6f8f8c8db19156aff2a9dde0e7f00dd58622870b2e7b3e49b79f2" -} diff --git a/.sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json b/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json similarity index 72% rename from .sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json rename to .sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json index 6f91ea852a0..27ff206084b 100644 --- a/.sqlx/query-52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e.json +++ b/.sqlx/query-74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT t.\"documentId\" AS document_id, t.metadata->>'markId' AS \"mark_id!\",\n array_agg(t.id ORDER BY t.id) AS \"thread_ids!\"\n FROM \"Thread\" t\n WHERE t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' ~ $1\n AND EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)\n GROUP BY 1, 2 HAVING count(*) > 1\n ORDER BY 1, 2 LIMIT $2", + "query": "SELECT t.\"documentId\" AS document_id, t.metadata->>'markId' AS \"mark_id!\",\n array_agg(t.id ORDER BY t.id) AS \"thread_ids!\"\n FROM \"Thread\" t\n WHERE t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' ~ $1\n AND EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id AND NOT p.\"wasDeleted\")\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)\n GROUP BY 1, 2 HAVING count(*) > 1\n ORDER BY 1, 2 LIMIT $2", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "52a2ecb544e678334f0b7b3292baa081cc95ce28236cf6a7aa97b37556caa98e" + "hash": "74842f5645a90d4bdefd392a080062648d83a046b36d7d83ed963c3842657a41" } diff --git a/.sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json b/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json similarity index 72% rename from .sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json rename to .sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json index 09728c3896a..62ee47591e2 100644 --- a/.sqlx/query-1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d.json +++ b/.sqlx/query-c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE t.\"documentId\" = ANY($1) AND t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' IS NOT NULL\n AND t.metadata->>'markId' !~ $2\n AND t.metadata->>'markId' NOT LIKE $3\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id)\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)", + "query": "SELECT count(*) AS \"count!\" FROM \"Thread\" t\n WHERE t.\"documentId\" = ANY($1) AND t.\"deletedAt\" IS NULL\n AND t.metadata->>'markId' IS NOT NULL\n AND t.metadata->>'markId' !~ $2\n AND t.metadata->>'markId' NOT LIKE $3\n AND NOT EXISTS (SELECT 1 FROM \"PdfPlaceableCommentAnchor\" p WHERE p.\"threadId\" = t.id AND NOT p.\"wasDeleted\")\n AND NOT EXISTS (SELECT 1 FROM \"PdfHighlightAnchor\" h\n WHERE h.\"threadId\" = t.id AND h.\"deletedAt\" IS NULL)", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "1134e40d502bcd585ed217d83b16c86de0e00f6bce372381114a12453714a39d" + "hash": "c32423d596dca1eb14bef93e22d6b2a37d52cfbf3cf650d1965df042056ad9ce" } diff --git a/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json b/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json new file mode 100644 index 00000000000..e0b1610c7fa --- /dev/null +++ b/.sqlx/query-c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO comms_message_threads (\n root_id, parent_entity_type, parent_entity_id, user_id, resolved, anchor,\n import_metadata, created_at, updated_at, deleted_at\n )\n SELECT tm.root_id, 'document', t.\"documentId\", t.owner, t.resolved,\n CASE\n WHEN pa.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_placeable', 'anchor_id', pa.uuid)\n WHEN ph.uuid IS NOT NULL\n THEN jsonb_build_object('type', 'pdf_highlight', 'anchor_id', ph.uuid)\n WHEN t.metadata->>'markId' ~ $3\n THEN jsonb_build_object('type', 'markdown', 'mark_id', (t.metadata->>'markId')::uuid)\n END,\n t.metadata, t.\"createdAt\" AT TIME ZONE 'UTC', t.\"updatedAt\" AT TIME ZONE 'UTC',\n CASE\n WHEN t.\"deletedAt\" IS NOT NULL THEN t.\"deletedAt\" AT TIME ZONE 'UTC'\n WHEN NOT EXISTS (SELECT 1 FROM \"Comment\" c WHERE c.\"threadId\" = t.id)\n THEN t.\"updatedAt\" AT TIME ZONE 'UTC'\n END\n FROM \"Thread\" t\n JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfPlaceableCommentAnchor\" a\n WHERE a.\"threadId\" = t.id AND NOT a.\"wasDeleted\"\n ORDER BY a.uuid LIMIT 1\n ) pa ON true\n LEFT JOIN LATERAL (\n SELECT a.uuid FROM \"PdfHighlightAnchor\" a\n WHERE a.\"threadId\" = t.id AND a.\"deletedAt\" IS NULL\n ORDER BY a.uuid LIMIT 1\n ) ph ON true\n WHERE t.\"documentId\" = ANY($1)\n ON CONFLICT (root_id) DO UPDATE SET\n user_id = EXCLUDED.user_id,\n resolved = EXCLUDED.resolved,\n anchor = EXCLUDED.anchor,\n import_metadata = EXCLUDED.import_metadata,\n created_at = EXCLUDED.created_at,\n updated_at = EXCLUDED.updated_at,\n deleted_at = EXCLUDED.deleted_at\n WHERE (comms_message_threads.root_id = ANY($2)\n OR comms_message_threads.updated_at <= EXCLUDED.updated_at)\n AND (comms_message_threads.user_id, comms_message_threads.resolved,\n comms_message_threads.anchor, comms_message_threads.import_metadata,\n comms_message_threads.created_at, comms_message_threads.updated_at,\n comms_message_threads.deleted_at)\n IS DISTINCT FROM\n (EXCLUDED.user_id, EXCLUDED.resolved, EXCLUDED.anchor, EXCLUDED.import_metadata,\n EXCLUDED.created_at, EXCLUDED.updated_at, EXCLUDED.deleted_at)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c5e01d489c1b1316ab38953bf81a99b5d98b5f73d5a4ce47c5499d9bf958ce6e" +} diff --git a/crates/macro_db_migrator/src/bin/comment_import/runner.rs b/crates/macro_db_migrator/src/bin/comment_import/runner.rs index a3ebd38590c..9b7c11394c5 100644 --- a/crates/macro_db_migrator/src/bin/comment_import/runner.rs +++ b/crates/macro_db_migrator/src/bin/comment_import/runner.rs @@ -148,7 +148,7 @@ async fn run_locked( async fn preflight(connection: &mut PgConnection) -> Result<(), rootcause::Report> { let ambiguous = sqlx::query_scalar!( r#"SELECT t.id FROM "Thread" t - LEFT JOIN "PdfPlaceableCommentAnchor" p ON p."threadId" = t.id + LEFT JOIN "PdfPlaceableCommentAnchor" p ON p."threadId" = t.id AND NOT p."wasDeleted" LEFT JOIN "PdfHighlightAnchor" h ON h."threadId" = t.id AND h."deletedAt" IS NULL WHERE t."deletedAt" IS NULL GROUP BY t.id HAVING count(DISTINCT p.uuid) + count(DISTINCT h.uuid) > 1 @@ -169,7 +169,7 @@ async fn preflight(connection: &mut PgConnection) -> Result<(), rootcause::Repor WHERE t."deletedAt" IS NULL AND t.metadata->>'markId' ~ $1 AND EXISTS (SELECT 1 FROM "Comment" c WHERE c."threadId" = t.id) - AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id) + AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id AND NOT p."wasDeleted") AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h WHERE h."threadId" = t.id AND h."deletedAt" IS NULL) GROUP BY 1, 2 HAVING count(*) > 1 @@ -277,6 +277,11 @@ async fn lock_legacy_tables(tx: &mut Transaction<'_, Postgres>) -> Result<(), sq sqlx::query(r#"LOCK TABLE "Comment", "Thread" IN SHARE ROW EXCLUSIVE MODE"#) .execute(&mut **tx) .await?; + // The 5s timeout bounds only the table-lock wait. Clear it so later batch + // statements wait normally for unrelated row locks instead of aborting. + sqlx::query("SET LOCAL lock_timeout = 0") + .execute(&mut **tx) + .await?; Ok(()) } @@ -487,7 +492,8 @@ async fn upsert_thread_state( FROM "Thread" t JOIN migrated_comment_thread_id tm ON tm.thread_id = t.id LEFT JOIN LATERAL ( - SELECT a.uuid FROM "PdfPlaceableCommentAnchor" a WHERE a."threadId" = t.id + SELECT a.uuid FROM "PdfPlaceableCommentAnchor" a + WHERE a."threadId" = t.id AND NOT a."wasDeleted" ORDER BY a.uuid LIMIT 1 ) pa ON true LEFT JOIN LATERAL ( @@ -689,16 +695,20 @@ fn remap_comment_ids(value: &mut serde_json::Value, mappings: &HashMap { - comment["id"] = serde_json::Value::String(message_id.to_string()); - stats.remapped += 1; + if let Some(id) = comment.get("id").and_then(legacy_comment_id) { + match mappings.get(&id) { + Some(message_id) => { + comment["id"] = + serde_json::Value::String(message_id.to_string()); + stats.remapped += 1; + } + None => stats.unmapped += 1, } - None => stats.unmapped += 1, } + // A comments array may nest inside a comment object. + let inner = remap_comment_ids(comment, mappings); + stats.remapped += inner.remapped; + stats.unmapped += inner.unmapped; } } else { let inner = remap_comment_ids(child, mappings); @@ -732,7 +742,7 @@ async fn count_invalid_mark_ids( AND t.metadata->>'markId' IS NOT NULL AND t.metadata->>'markId' !~ $2 AND t.metadata->>'markId' NOT LIKE $3 - AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id) + AND NOT EXISTS (SELECT 1 FROM "PdfPlaceableCommentAnchor" p WHERE p."threadId" = t.id AND NOT p."wasDeleted") AND NOT EXISTS (SELECT 1 FROM "PdfHighlightAnchor" h WHERE h."threadId" = t.id AND h."deletedAt" IS NULL)"#, documents, diff --git a/crates/macro_db_migrator/src/bin/comment_import/test.rs b/crates/macro_db_migrator/src/bin/comment_import/test.rs index 85b1296483f..43a2e14943d 100644 --- a/crates/macro_db_migrator/src/bin/comment_import/test.rs +++ b/crates/macro_db_migrator/src/bin/comment_import/test.rs @@ -537,12 +537,14 @@ async fn rejects_concurrent_runner_ambiguous_anchors_and_duplicate_marks(pool: P assert!(mappings(&mut connection).await.is_empty()); // The lock is released after a failed preflight. + let mut probe = other_connection(&pool).await; let acquired: bool = sqlx::query_scalar!(r#"SELECT pg_try_advisory_lock($1) AS "acquired!""#, LOCK_ID) - .fetch_one(&mut other_connection(&pool).await) + .fetch_one(&mut probe) .await .unwrap(); assert!(acquired); + probe.close().await.unwrap(); sqlx::query!(r#"UPDATE "Thread" SET metadata = '{"markId":"not-a-mark"}' WHERE id = 5"#) .execute(&mut connection) @@ -665,18 +667,27 @@ fn pdf_comment_ids_are_remapped_at_any_depth_and_unmapped_ids_are_counted() { let mapped_id = Uuid::from_u128(12); let mappings = HashMap::from([(12_i64, mapped_id)]); let mut payload = json!({ - "placeables": [{"payload": {"comments": [{"id": "12", "content": "a"}, {"id": "13", "content": "b"}]}}], + "placeables": [{"payload": {"comments": [ + {"id": "12", "content": "a"}, + {"id": "13", "content": "b"}, + {"id": "wrapper", "comments": [{"id": 12}]} + ]}}], "highlights": {"1": [{"thread": {"comments": [{"id": 12}, {"id": "external"}, "not an object"]}}]}, "comments": "not an array", }); let stats = remap_comment_ids(&mut payload, &mappings); + // Two top-level 12s plus the 12 nested inside the "wrapper" comment. assert_eq!( stats, RemapStats { - remapped: 2, + remapped: 3, unmapped: 1, } ); + assert_eq!( + payload["placeables"][0]["payload"]["comments"][2]["comments"][0]["id"], + mapped_id.to_string() + ); assert_eq!( payload["placeables"][0]["payload"]["comments"][0]["id"], mapped_id.to_string() diff --git a/docs/legacy-comment-import.md b/docs/legacy-comment-import.md index bfebdda6e91..02b2b8eac22 100644 --- a/docs/legacy-comment-import.md +++ b/docs/legacy-comment-import.md @@ -36,7 +36,7 @@ Two runs cannot overlap: a session advisory lock makes the second exit with writes: set `LEGACY_COMMENT_WRITES_ENABLED=false` on the document storage service (Doppler project `cloud-storage-service`) and roll it out. Comment create, edit, and delete and anchor delete then answer `503` with - `Document comments are read-only while they move to the new message store`. + `Document comments are read-only while they move to the new message store. Refresh the app in a few minutes and try again.`. Reads, highlight creation, and anchor edits keep working. 3. Run the importer once more. It must end with `Nothing to import` on a second invocation.