From 0cabada3a1b67278f072a942ea6ab2684c537086 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 14:54:25 +0000 Subject: [PATCH] feat(upload): upgrade legacy .doc/.ppt/.xls uploads to OpenXML Uploaded legacy binary Office files are converted on the server so they open in the native surfaces: .doc -> .docx (then the DOCX pipeline), .ppt -> .pptx (presentation editor) and .xls -> .xlsx (uploaded-workbook preview, importable into a native spreadsheet). - The upload finalizer asks the convert service to write upgraded.{docx,pptx,xlsx} when a legacy original lands, keeping the original ready so it stays downloadable if conversion fails. - When the upgraded object lands, the finalizer makes it the document's latest version (a DocumentInstance, or a DOCX BOM staged for the DOCX pipeline) and swaps the file type with a compare-and-swap, rolling back if the copy fails. Redelivered events are no-ops. - The convert service supports doc->docx, ppt->pptx and xls->xlsx, accepts explicit file types for extensionless versioned keys, gives upgrades 120s, and exits its LibreOffice child without teardown, which could crash Writer after a successful save. - The file view shows a converting state for fresh legacy uploads and reopens the document in its new block when the upgrade lands. - Infra: finalizer env, S3/SQS permissions and docx bucket policy. Tests cover the domain flow, the repository queries, finalizer routing, key parsing, real LibreOffice conversions (ignored without LOK_PATH), and the docx/pptx engines and spreadsheet importer on LibreOffice output. --- .github/workspace-dep-closures.json | 1 + ...bd5e309aab299895ab0e4a91c01a870d9109c.json | 16 + ...42b63e816133ecea6ce72c8c9507277617f30.json | 22 + ...0395eb184c9eb259894f689390b13e5647a6c.json | 14 + ...944cdab75327b002a16e85cf9da6fe8f18c8c.json | 14 + ...09f2d94ad20d9f6e193e1e92655d335a128d3.json | 23 + Cargo.lock | 8 + .../core/xlsx-fixtures/README.md | 2 + .../core/xlsx-fixtures/upgraded-from-xls.xlsx | Bin 0 -> 7832 bytes .../core/xlsx-upgraded-legacy.test.ts | 122 ++++ .../views/SpreadsheetEditor.tsx | 9 +- .../block-unknown/component/Block.tsx | 74 ++- .../component/UnknownContent.test.tsx | 38 ++ .../component/UnknownContent.tsx | 30 +- .../block-unknown/legacy-office.test.ts | 178 ++++++ .../features/block-unknown/legacy-office.ts | 89 +++ crates/documents/src/domain.rs | 4 + .../src/domain/legacy_office_upgrade.rs | 336 +++++++++++ .../src/domain/legacy_office_upgrade/tests.rs | 556 ++++++++++++++++++ .../src/outbound/pg_document_repo.rs | 1 + .../pg_document_repo/legacy_upgrade.rs | 113 ++++ .../pg_document_repo/legacy_upgrade/tests.rs | 205 +++++++ .../tests/fixtures/upgraded-from-doc.docx | Bin 0 -> 17208 bytes crates/docx_engine/tests/upgraded_legacy.rs | 132 +++++ crates/model/src/convert.rs | 9 + .../tests/fixtures/upgraded-from-ppt.pptx | Bin 0 -> 58348 bytes crates/pptx_engine/tests/upgraded_legacy.rs | 115 ++++ crates/s3_key/src/document_key.rs | 100 +++- crates/s3_key/src/document_key/test.rs | 47 ++ crates/s3_key/src/lib.rs | 6 +- docs/AGENT_GUIDE/documents.md | 25 +- .../document-upload-finalizer-lambda.ts | 45 +- .../docx-unzip-bucket.ts | 16 + infra/stacks/cloud-storage-service/index.ts | 7 + services/convert_service/Cargo.toml | 5 + .../fixtures/legacy/budget.xls | Bin 0 -> 8192 bytes .../convert_service/fixtures/legacy/deck.ppt | Bin 0 -> 669184 bytes .../convert_service/fixtures/legacy/memo.doc | Bin 0 -> 29696 bytes .../src/api/backfill/backfill_docx.rs | 2 + .../convert_service/src/process/convert.rs | 87 +-- services/convert_service/src/process/test.rs | 355 +++++++++++ services/convert_service/src/utils.rs | 67 ++- .../Cargo.toml | 5 + .../src/app.rs | 57 +- .../src/app/tests.rs | 365 ++++++++++++ .../src/outbound.rs | 219 ++++++- .../src/outbound/tests.rs | 103 ++++ .../src/runtime.rs | 32 +- .../src/service/document/process_document.rs | 2 + services/search_upload_handler/src/handler.rs | 3 + 50 files changed, 3577 insertions(+), 82 deletions(-) create mode 100644 .sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json create mode 100644 .sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json create mode 100644 .sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json create mode 100644 .sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json create mode 100644 .sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json create mode 100644 apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx create mode 100644 apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts create mode 100644 apps/web/src/features/block-unknown/component/UnknownContent.test.tsx create mode 100644 apps/web/src/features/block-unknown/legacy-office.test.ts create mode 100644 apps/web/src/features/block-unknown/legacy-office.ts create mode 100644 crates/documents/src/domain/legacy_office_upgrade.rs create mode 100644 crates/documents/src/domain/legacy_office_upgrade/tests.rs create mode 100644 crates/documents/src/outbound/pg_document_repo/legacy_upgrade.rs create mode 100644 crates/documents/src/outbound/pg_document_repo/legacy_upgrade/tests.rs create mode 100644 crates/docx_engine/tests/fixtures/upgraded-from-doc.docx create mode 100644 crates/docx_engine/tests/upgraded_legacy.rs create mode 100644 crates/pptx_engine/tests/fixtures/upgraded-from-ppt.pptx create mode 100644 crates/pptx_engine/tests/upgraded_legacy.rs create mode 100644 services/convert_service/fixtures/legacy/budget.xls create mode 100644 services/convert_service/fixtures/legacy/deck.ppt create mode 100644 services/convert_service/fixtures/legacy/memo.doc create mode 100644 services/document_upload_finalizer_handler/src/app/tests.rs create mode 100644 services/document_upload_finalizer_handler/src/outbound/tests.rs diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json index f5680947d89..4e731e7c62c 100644 --- a/.github/workspace-dep-closures.json +++ b/.github/workspace-dep-closures.json @@ -4877,6 +4877,7 @@ "crates/macro_env_var", "crates/macro_event_broker", "crates/macro_event_topics", + "crates/macro_queues", "crates/macro_service_urls", "crates/macro_sync_service_jwt", "crates/macro_tower_layers", diff --git a/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json b/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json new file mode 100644 index 00000000000..1b5d58ed199 --- /dev/null +++ b/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \"Document\"\n SET \"fileType\" = $3,\n \"updatedAt\" = NOW()\n WHERE id = $1 AND LOWER(\"fileType\") = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c" +} diff --git a/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json b/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json new file mode 100644 index 00000000000..585499ecb11 --- /dev/null +++ b/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO \"DocumentBom\" (\"documentId\", \"createdAt\", \"updatedAt\")\n VALUES ($1, NOW(), NOW())\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30" +} diff --git a/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json b/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json new file mode 100644 index 00000000000..c8b812d4563 --- /dev/null +++ b/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM \"DocumentInstance\" WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c" +} diff --git a/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json b/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json new file mode 100644 index 00000000000..ee430a934f7 --- /dev/null +++ b/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM \"DocumentBom\" WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c" +} diff --git a/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json b/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json new file mode 100644 index 00000000000..208428c2618 --- /dev/null +++ b/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO \"DocumentInstance\" (\"documentId\", \"sha\", \"createdAt\", \"updatedAt\")\n VALUES ($1, $2, NOW(), NOW())\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3" +} diff --git a/Cargo.lock b/Cargo.lock index c2f4f11447e..4e7a4336e5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4802,6 +4802,7 @@ dependencies = [ "aws-sdk-sqs", "axum", "database_env_vars", + "docx_engine", "doppler-config", "futures", "http-body-util", @@ -4820,6 +4821,7 @@ dependencies = [ "macro_uuid", "model", "nix 0.29.0", + "pptx_engine", "remote_env_var", "rs-libreoffice-bindings", "s3_client", @@ -4830,6 +4832,7 @@ dependencies = [ "sqlx", "sqs_client", "sqs_worker", + "tempfile", "tokio", "tower 0.5.3", "tower-http", @@ -6396,12 +6399,17 @@ dependencies = [ "macro_aws_config", "macro_entrypoint", "macro_env_var", + "macro_queues", "macro_service_urls", + "macro_uuid", "model", + "model_owner", "s3_key", "serde", "serde_json", + "sha2 0.10.9", "sqlx", + "sqs_client", "sync_service_client", "tokio", "tracing", diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md index d494582b0dd..edd7863d15b 100644 --- a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md @@ -49,3 +49,5 @@ Notices: - U.S. federal works (SEC, Census, BLS, BEA, IRS, SBA, GSA) are in the public domain; the FRED export carries BLS series data. Templates whose terms forbid redistribution (Microsoft Create, Vertex42, Breaking Into Wall Street, Damodaran, SBDC) were used only for local testing and are not committed. Add a file by copying it here unmodified and adding its `file`, `title`, `source` and `license` to the manifest; set `skipCalculation` with a reason only when the engine cannot finish it. + +`upgraded-from-xls.xlsx` is a legacy Excel 97 workbook (`services/convert_service/fixtures/legacy/budget.xls`) converted to `.xlsx` by LibreOffice with the `Calc MS Excel 2007 XML` filter, exactly as the convert service upgrades uploaded `.xls` files. `../xlsx-upgraded-legacy.test.ts` checks that it imports into a native spreadsheet with its sheets, formulas, merges and number formats. diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..775d52de0ae1fdf29b35cf767be872a6344ee5c8 GIT binary patch literal 7832 zcmbVxby!sI7A_4U64EUx9fAl5BHi5`GXn$C-AH$X#30=*Dd3=_lync>0s~4Tb-{C< zBgcEs?~l9oGx^L~v-kVG`(5i>OGO?Tg#ZBq0|No-C}oIn&&Y1yyIXS_JA$m8INj_V z-@mqs^b(733WB7zJ?Vde0Yg+8QEwD zTUZKwQAH=%ic2GKZ1!w?oQCy`Bi1aQfhR77w2y(5mKrVUX&6A5*smUV*;3?F7v@cS zmi%~biu;0i>IO@Fux*Y_kfiJZ#GbE|- zN$8eD5=r7D^yn^pO7lox%wYLW`9&v3bD-V#i;zm%VDN1BZvSEP3)4)=N=J_RfYkSG zHIS~Bb5)38>G5EmHwd6bJIS-ysQZAh(Z?{5RyjwpE56cv2R4f^Wb@!{@=@Lsk_$rO zhU>Aif|~qq|J<(5yk^f8*!?t9cd#xcu;@TdE|@Ptmu>j z>&B`n0?OI|`G@1+lGv=#syG^#&H|v(AYQvA*RGgv;IK@*Mo3ijvgDVM>w_2Cq3%`D zXN820O}$a?o|%JBQmiBhBu`!p6_SqgDBD;FK{1ne00Usj&9cK0N@K|w26lU)RIHaU z;V_$UZb;Brm`Bf8k+}*tIcky?24IBHfhj{!0Ab>d?RcEuL_AFQ^f-xp3;t>miBF^V zBu*Vvgik*usuM3~W^Eoou|Y%48#Xl)%PZ)WOmRi$v9;w)0oEdASAIegp?qY7cT;WI z^ZKXiCdvnT7UCP2Ip-;%%-XoA2u474TQI(F^;Rn}rJUlXNF9@;RkmgpKJDR&ROjx+ zg#)h=<7zQ6dxt0|*1i-J)@DDhzgRKFdjd>Wm=cjm!gi5T9zAFTZ{c_am^m?;X6X5I zP%>oY&Nc!z+&uLzf77UQV;_i#gn&@~k2J#n7mb|3AR7?p?`Q72yKX*N(GJFi6|#Rp z7_e8i^6|mw6D4l8hptT3;tlhpX$=!HH8dsggZ-i&rIo6j4la`|?{FDmZ^?)#X!A`D zy@wP)vIL)OUV%yUTffGZP|jkj?II`B2W)4|n(@?OIz7+NzKsC|tSU4=muER>a)O;E z61e-JHsi9)VJsnfDIVb8U1X)=I3eJY?AfaCb%*&`G zk%KP|oq5^t^tj2rgZ7ny=(DXk0Mgu&u*h!Iid2g$!}Lm8a)<_wJ0hJ?E7r#ZFi-nX(?7Q$c&hF zF@)&Gh~)$-hrMOrCTN#+2~DyGr=KSJLi)=61X4jg^PEE3WBU`dc`bg_=72Q#A^b+* z^Hsafrd>Z8^+~P875<${omBq}pt_~k2*y9D)T4jV%gNcp8sv0GEV!4tK2jrpp4l@)xGE=f(Ng1nt9|c%~*$V9PUR z5WK+q>l!8sw3nx7@mUvp2I5eDq(@=#Wdtcv7}6q{5HkUR5 z8I;U}{@mp$$J+Zmxh%aBp+y;v%i(6VNU}(fJmzpY>vl7VUMXCg0ae+pw$!3W?vWA| z1kcaYw+p*v3lfv;P`M34A1nvAz38Ut!hW-;Q=cPLpoXq{D6np+0?F`glfyLM_vZsrKCB%>hT#m?a-Tn&Rm; z_yEW8WFkG0Y&|4WfqmLG1AjQ}Q&e^UkA#R8YaE>(U>LG@PCQ9EB_H8b(ubjDeXM&R zSG%lspl0YyqSy)CNL z0EZ*KH<#ZBuh>(i+eo6lrm*^8LSJ;c2t6}yp#oUyNk+B){b_~^eIvd4uxS40A?2h1 z>GzDA+v5j9gjQfGmWledei|NVfi=5~2GH+1H6oq8TO$Yn)tKed+Jps{l3eKxK^a_s}}Z%A+tYa|RWfv=EI zl-`~aU$&qJ)2V<_;Xz;6gpX2N@XSvL%r)L{x6jYm?|+b)pg4*Hqj0|rljepg0$e85 z8^7w#Eku~%!x|Hng*U3b3`j7JtLF23Dg`;vIHDS0!VfpypBV59&sq#^w6kT9*fe^X z75f>u?54_4p2SP3;s)i&kMK8m#K1a>GP#0=lfQK-((Sse=`7EW6 z6$y4eS95#XEaf@p(de<;{GBdhw-#Os@IDc&T;P5Mkv>hL#7FVeGMqDDorZ_grbwUW zLZv%~bCinxb`Mwe7({U;U1PhaNm2a!^R)9|t7h2(AB98@GY^zA7ky6+PGfJ_4R4U| zRHBnfW@P6!0Xb6slS+{Msb!qNAds^Y=kJ%_nFyMyXt&CT8?t{z7@!1ii)YfO#O9^c34t8FSr+{Lh7@j{A$y&(jtN)T>gtZzp5D*Q;ir#Cs;0?uQ{#iAAQvvEEZ_PV@R3&QZNX1@3d*eEXg^>6%!R za?Pf!g{{fQ&A(V$-{$DM67dvWxpT`4WQ4{HBv=bKiX_0$SKi20oIz^EpSw4dR`Ab> zzio-qdjEO$O!nX?o*1iKZclI3cm+3d7t!SPdu-xs)7KR|GOaUWVX@W~o+%Nps&u*^ zca1K)DZfl^V=SAx)CEEVlPbP*s0*S@EA+vrh?K2#~L12Tp?eg zxsF(D(f&vmjdE4&z%R45JGSCqybNXpHJ)FI{4jhj@>P zPoUG*g8o6S=w;6d8?p;!-!NJ$WFRB40w(iLW28$VA`rDGn{=Or;XrEKZUo-CZrK;H zFbDQ~tVH=RLrzKxET=UiG+ z550d8;eXL={^FIdSG&2lCLf@ak!w1NoBSgI7M~7*f&yP*w+Rnb*J_NSz5lnhQftOA zMR0XmhH2n7T4cZ%AixTXFHu$K`?LaQ_0OulZfR@U!06Z+_hQ>>rm)}6tFdOEL91TG z(?B$4u= zgnKC{kAHVbf_OI5}00J^|?O#)9*$+T0gj4Fqvl1 zC_?{Hl-~Sw&Z?XEvA)m6<<5Jzhv!$5jMA6PQq%ltmz}b^^WD-<@mlv5mjU1BIbvUl z1yuT2a546z&5h&7Vj{pXA5Bx~K*GnNnKBbMq}1z?y38IqnSGv~U>nj2z|iVzD<=x6 zw0DH+^X_0)dO!T&T-^c8uC<@eM+TZdQ8@X(f#E#KVYZrSP@2tr0zCJO-pPEOS@l>^FdA$gZuL)D3NY3A-H(sQ1QLJGKcy$y_ zfjyKV&_T+4b*9s|G*FM1JxQM_LBGU;%s6hT*gC)eicQvpO*2YP#al7+BMtQnkpkjj zoWi9!G>q(lbb?8(DN}GwK+3p6_6nx5<1j=0$BCk0jdb-5&g5qbxnR6^8cd^440cXU zUS~!|+0m5AS6U8$mzeY^akIPbh=v}xP_?bUqlC+MWu;)k2Wg)QruGTCh8z->1 z$o0icD!}6HL7ZwhOP4C|?9E=)=LSLH{Vn`)qm+~J`boT{wloUcwbHdQ`(Ik)^``v+ zCmGS>c(+eb6Hj%p(w@lCKDy`RpB2sqB|b=oMS(tBx)zuH_$oC78oQZg^$<#3Nk+} zCreZn#~Kneubr8IW1ZA%@f$_#97gN+g5_cgT^3=7))); zoZZ#~aRH`k2ZuQ9XkW7Q%X-vM#VHq5@K3BhHIK-|%pKK-tDAihyKvEIg>p`fEEKeN zz6279Wgagnn;-GT`e{m6Gc_6$`8M;1dQfn#^2-%Q(ubRC)Au0m#>Te$icJ!pfVz%F zg#b>KDr+NHKN4$g@;}a>%!?zTf0%kD7#P{i`EeEyqT|laC3<72pNLsEp<(isn&BJO zE9Mg@ZUc2UqHm8>AZN$*=ipHvx@dC4?ci*fTzT)f>P`{`8>vRSW_N6)RPuY`CnmH* zsx}}uF$j%@-2%@^zlZw(gn;Bt2;Zxi5wes=W`Mn^Rg;f!KQ6wT+x`+d#-#E!DDAdJ z%m2rn9KpYLa$r+O5K!IO(bCr9_l7QctWBwd3kzZRr7_-mYXLh~1M@fRL|4e`v9q_u z$FnjO>Sm2!7V-JNJYx9DKh!|`_16z>Y%~AYg8=q^7aCNPRsB#ycH^p|2`x;s$O=j! zF?JFEwYE7+S|V=3l6S&~b=nc>R8sP9L=d|n5S+YqcdYlh%x`T<(3c!kG*zv~!IW|% zXQWN>{P3Yq-5|kfRTFh6LJ5a3|0v)QxAwA*luzj|x9x`t}ZP?5ni3Q- zJ>)8rFtnd~_Eia+ozGI=;lu7_mWIROH{v@OA<|ZIsE7y%FHrtTB_H0x_^&4C@5W|D z!f*6K_PrulPM^=e#>gpbQIKh&;idXQZ%bYup4S<8a3X>K7%RG_BgVnk@7kbV^vVY@u-yk@hH+}CGgu&b1!T;`V;@$oNXa`Vn zw6k~O1lT!(?y|^xwfGEY@Huf!&xmU+mYs?TTXM#``XGvWli>gZkU~NE zaq5#1kubu#cT$k2;` znifk*J2$LZ*J+>|x}s&6GUXsyK2pj>@v~&#>ND^j=3Y!nuC54(9QCJ|cuFT_I>xKY z1JAF#LHS%xH8o$H6X&amh1)wAov(mg{{`71?qnTp!^88YT9@`&_|rqK@DBG@S#wKo zh|!P{&*cvE#qO#e{%4hmOt~sDQO+fkRHD9S{e+lH#E+lyYxqBQ;m0kz&dW;mWYy(& zSI+%(Lpn3u5#lfrx0oQkI0XlaYU_5Wo{7k;RBomIs{FPLr%Z)Qli?HVo62u3(_9{M z_CCGiF|5St!0eXCtN*w_xc3*x)ZYI0`oy)_w%;`kZ^^te%P2u9&h%)O7amkcDOjhU z%)kM(N-TCz^|(MCetYO$c}ryW4ueAr7)zp+Bn1sM>Y}YT+^+N8A}n;AR$Xk+>L{B6;9xUb zSc{s9*`p~?Rlatlsl+}rF8=w;*?4a9Yn;2<_;9Eajf|+BC0x~|9xhr#Z~X8lg_uQs z`&!uC^giOk5GxbsLn9$V?}iVy8R=E)xw=U+^Ad@^XNb{tYt|>vW(U^{hP~#(q0m(J z%~(`<2dc9%6zDmNAL1QwM(+?~FK%JUxCQTjGyAUL`iI#rPR@2Vzp)$qfZ5JP5WGBJ zgfRotK#w1-i{yW)aT>SKfaqk`qZsMqMHjFa6H5I`X-xP^yc~B1&#H-&l^KEk5%yDd zyQ>0z*|y##6RM!>bu_jX*C_)!)UaN}NVX?$1d0f(3Z@5_9;FBXIOrtp)G{5^cw+|J z({@)769qLURA++m&uP5{&VOz{zmmrWF7BT#*wJ{|!n=Dz4656+=${%)Nil9eQ;>?GcWpO&9$xnKkFf)sU zmQ?0Iu(1Gp)2*_m;usZ;su(~1F~Q5zg|yaSzyr;Cdf)h&X&pM8&e$Pnb&W`^mC4(_ zF&8eCvSeyMwx+3;7pJe!)FvFPO9=9Ez{V6pGO%h|<@n`HgkLk*W}y3tzud&7fiAnR zw8qL$WaV4CuMFjr0lIn^^Mkgemp5PJ24lMI~a-c7Ydk zQPKKL{};)1BF;b)^7$HNG^Exq&SnqrDn0U)I3+qm>HP;8x)IR}$InMB&cJYt4DGK& zb5ileFHea}X`YHUOb|UhD%@pT9KIyjXDGiGju!cy*+0g;&-J)v<2SUsl|w`#K=?C! zyFX>R8@~Ow{qIrS-*N7b8SZ9Me~kO~pKs&bpHThne7}i&cVql9_uGQx*7@(b#NPq# z_m}Q^&3|m_cEfTT;7{lIZ`b>~n!5(nAG><+pA-MT+EIT;xnDf}ca-*9H2!`0_oMvY z5B(kGe(i8qX#KIATUhQcRpZT$F^<*-m5tNcDyh5cUkw3;Sk+ds(Dzt{i!EtvoD2K+}R{|<1U;r|_=jT8al|Di`J W@~F2r0|Elp?SX#FF)#AFPyYwv#GwuV literal 0 HcmV?d00001 diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts new file mode 100644 index 00000000000..13f4282bac4 --- /dev/null +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts @@ -0,0 +1,122 @@ +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { initSync } from '@ironcalc/wasm'; +import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation'; +import { + importSpreadsheetSheets, + readSpreadsheetWorkbook, +} from '@macro-inc/spreadsheet/workbook-document'; +import ExcelJS from 'exceljs'; +import { LoroDoc } from 'loro-crdt'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { decodeXlsx, encodeXlsx } from './xlsx-codec'; + +/** + * An uploaded legacy `.xls` is upgraded to `.xlsx` by LibreOffice on the + * server; this is that output for a two-sheet budget. It must import into a + * native spreadsheet like any Excel-written workbook. + */ +const upgraded = () => + new Uint8Array( + readFileSync( + createRequire(import.meta.url).resolve( + './xlsx-fixtures/upgraded-from-xls.xlsx' + ) + ) + ); + +beforeAll(() => + initSync({ + module: readFileSync( + createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm') + ), + }) +); + +describe('workbook upgraded from a legacy .xls', () => { + it('imports both sheets with their cells and formulas', async () => { + const imported = await decodeXlsx(upgraded()); + + expect(imported.sheets.map((sheet) => sheet.name)).toEqual([ + 'Budget', + 'Notes', + ]); + const [budget, notes] = imported.sheets; + expect(budget.cells.A1.value).toBe('Month'); + // Stored as text in the .xls, so it stays text rather than becoming a date. + expect(budget.cells.A2.value).toBe("'2026-01"); + expect(budget.cells.B2.value).toBe('1050'); + expect(budget.cells.D2.value).toBe('=B2-C2'); + expect(budget.cells.D13.value).toBe('=B13-C13'); + expect(budget.cells.A15.value).toBe('Total'); + expect(budget.cells.B15.value).toBe('=SUM(B2:B13)'); + expect(budget.cells.D15.value).toBe('=SUM(D2:D13)'); + expect(budget.cells.F1.value).toBe('Merged header'); + expect(notes.cells.B1.value).toBe('Finance'); + expect(notes.cells.B2.value).toBe('=Budget!D15/Budget!B15'); + }); + + it('keeps header styles, number formats and the merged header', async () => { + const imported = await decodeXlsx(upgraded()); + const budget = imported.sheets[0]; + + expect(budget.cells.A1).toMatchObject({ + bold: true, + fillColor: '#DDEBF7', + }); + // `#,##0.00` imports as Macro's thousands-separated number format. + expect(budget.cells.B15).toMatchObject({ format: 'number', decimals: 2 }); + expect(budget.metadata?.merges).toEqual(['F1:H1']); + expect(imported.warnings).toEqual([ + 'Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.', + ]); + }); + + it('calculates the same results after native save and reopen', async () => { + const imported = await decodeXlsx(upgraded()); + const doc = new LoroDoc(); + const reopened = new LoroDoc(); + const engine = createInitializedSpreadsheetCalculator(); + try { + importSpreadsheetSheets(doc, imported.sheets, true); + reopened.import(doc.export({ mode: 'snapshot' })); + const sheets = readSpreadsheetWorkbook(reopened); + const results = engine.calculateWorkbook( + sheets.map((sheet) => ({ ...sheet, rowCount: sheet.layout.rowCount })), + { includeTypes: true } + ); + + const budget = results[sheets[0].id]; + // Revenue 1000 + 50i and cost 600 + 20i for months 1..12. + expect(budget.B15.number).toBe(15_900); + expect(budget.C15.number).toBe(8_760); + expect(budget.D15.number).toBe(7_140); + expect(budget.D2.number).toBe(430); + expect(budget.B15.display).toBe('15,900.00'); + expect(results[sheets[1].id].B2.number).toBeCloseTo(7_140 / 15_900, 12); + for (const values of Object.values(results)) { + expect( + Object.entries(values).filter(([, value]) => value.error) + ).toEqual([]); + } + } finally { + engine.dispose(); + } + }); + + it('exports back to an .xlsx Excel reads with the same formulas', async () => { + const imported = await decodeXlsx(upgraded()); + + const exported = await encodeXlsx({ sheets: imported.sheets }); + + const external = new ExcelJS.Workbook(); + await external.xlsx.load(exported.bytes.slice().buffer); + const budget = external.getWorksheet('Budget')!; + expect(budget.getCell('B15').formula).toBe('SUM(B2:B13)'); + expect(budget.getCell('D2').formula).toBe('B2-C2'); + expect(budget.getCell('F1').isMerged).toBe(true); + expect(external.getWorksheet('Notes')!.getCell('B2').formula).toBe( + 'Budget!D15/Budget!B15' + ); + }); +}); diff --git a/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx b/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx index 42cdf952b13..99c0998a08c 100644 --- a/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx +++ b/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx @@ -529,9 +529,16 @@ export function SpreadsheetEditor(props: { async function importFile(file: File | undefined) { if (!file || !editable()) return; + if (/\.xls$/i.test(file.name)) { + // Converting .xls to .xlsx needs the server; uploads are upgraded. + actions.setNotice( + 'Legacy .xls workbooks are converted when uploaded. Upload the file, then choose Edit in Macro.' + ); + return; + } if (!/\.(csv|xlsx)$/i.test(file.name)) { actions.setNotice( - 'Choose a .csv or .xlsx file. Legacy .xls and macro-enabled workbooks are not supported.' + 'Choose a .csv or .xlsx file. Macro-enabled workbooks are not supported.' ); return; } diff --git a/apps/web/src/features/block-unknown/component/Block.tsx b/apps/web/src/features/block-unknown/component/Block.tsx index dc7355e4e43..897d74ae1fe 100644 --- a/apps/web/src/features/block-unknown/component/Block.tsx +++ b/apps/web/src/features/block-unknown/component/Block.tsx @@ -1,8 +1,10 @@ import { FileSidePanelSections, SidePanel } from '@components/app/side-panel'; +import { useSplitLayout } from '@components/app/split-layout/layout'; import { useBlockId } from '@core/block'; import { DocumentBlockContainer } from '@core/component/DocumentBlockContainer'; import { toast } from '@core/component/Toast/Toast'; import { useShareModal } from '@core/component/TopBar/shareModal'; +import { fileTypeToBlockName } from '@core/constant/allBlocks'; import { blockMetadataSignal } from '@core/signal/load'; import { useGetPermissions } from '@core/signal/permissions'; import { @@ -10,11 +12,26 @@ import { useBlockDocumentName, } from '@core/util/currentBlockDocumentName'; import { downloadFile } from '@filesystem/download'; +import { waitForDocumentContentReady } from '@queries/storage/document-location'; +import { fetchDocumentMetadata } from '@queries/storage/document-metadata'; import { createCallback } from '@solid-primitives/rootless'; -import { lazy, Show, Suspense } from 'solid-js'; +import { + createSignal, + lazy, + onCleanup, + onMount, + Show, + Suspense, +} from 'solid-js'; import { SpreadsheetSkeleton } from '../../block-spreadsheet/components/SpreadsheetSkeleton'; import { isUploadedWorkbook } from '../../block-spreadsheet/core/uploaded-workbook'; import { useSpreadsheetAccess } from '../../block-spreadsheet/primitives/use-spreadsheet-access'; +import { + legacyOfficeUpgradeTarget, + pollForLegacyUpgrade, + shouldAwaitLegacyUpgrade, + upgradedOfficeLabel, +} from '../legacy-office'; import { useGetFileBlob } from '../signal/blockData'; import { TopBar } from './TopBar'; import { UnknownContent } from './UnknownContent'; @@ -50,6 +67,7 @@ function BlockUnknownContent() { owner: blockMetadataSignal()?.owner, })); const getBlob = useGetFileBlob(); + const convertingTo = useLegacyOfficeUpgrade(blockId); const downloadDocument = createCallback(async () => { try { @@ -74,6 +92,7 @@ function BlockUnknownContent() { fallback={ void downloadDocument()} /> @@ -88,3 +107,56 @@ function BlockUnknownContent() { ); } + +/** + * Waits for a just-uploaded .doc/.ppt/.xls to be upgraded to OpenXML, then + * reopens it in the block for its new type. Returns the label of the format + * being converted to while waiting. + */ +function useLegacyOfficeUpgrade(documentId: string) { + const { replaceSplit } = useSplitLayout(); + const initial = blockMetadataSignal.get(); + const target = legacyOfficeUpgradeTarget(initial?.fileType); + const [waiting, setWaiting] = createSignal( + !!initial && !!target && shouldAwaitLegacyUpgrade(initial, Date.now()) + ); + + onMount(() => { + if (!target || !waiting()) return; + const controller = new AbortController(); + onCleanup(() => controller.abort()); + + void (async () => { + const upgraded = await pollForLegacyUpgrade({ + target, + fetchMetadata: () => fetchDocumentMetadata(documentId), + fileTypeOf: (metadata) => metadata.fileType, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + signal: controller.signal, + }); + if (controller.signal.aborted) return; + if (!upgraded) { + setWaiting(false); + return; + } + // Word content becomes readable once the DOCX pipeline processes it. + if (target === 'docx') { + await waitForDocumentContentReady({ + documentId, + timeoutMs: 60_000, + }).catch(() => undefined); + if (controller.signal.aborted) return; + } + const blockName = fileTypeToBlockName(upgraded.fileType); + if (blockName === 'unknown') { + // Excel workbooks stay in this block, which previews them. + blockMetadataSignal.set(upgraded); + setWaiting(false); + return; + } + replaceSplit({ content: { type: blockName, id: documentId } }); + })(); + }); + + return () => (waiting() && target ? upgradedOfficeLabel(target) : undefined); +} diff --git a/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx b/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx new file mode 100644 index 00000000000..25d6483c16c --- /dev/null +++ b/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx @@ -0,0 +1,38 @@ +import { cleanup, render, screen } from '@solidjs/testing-library'; +import { afterEach, expect, it, vi } from 'vitest'; +import { UnknownContent } from './UnknownContent'; + +afterEach(cleanup); + +it('offers the file when there is no preview', () => { + render(() => ( + + )); + + expect(screen.getByText(/No preview available for/)).toBeTruthy(); + expect(screen.queryByTestId('legacy-office-converting')).toBeNull(); + expect(screen.getByText('Download')).toBeTruthy(); +}); + +it('says a legacy Office file is being converted and still offers the original', () => { + const onDownload = vi.fn(); + render(() => ( + + )); + + const status = screen.getByTestId('legacy-office-converting'); + expect(status.textContent).toContain('Board Update.ppt'); + expect(status.textContent).toContain('to a PowerPoint presentation'); + expect(screen.queryByText(/No preview available/)).toBeNull(); + screen.getByText('Download').click(); + expect(onDownload).toHaveBeenCalledOnce(); +}); diff --git a/apps/web/src/features/block-unknown/component/UnknownContent.tsx b/apps/web/src/features/block-unknown/component/UnknownContent.tsx index 98c9ee2c75e..64f44dfae2c 100644 --- a/apps/web/src/features/block-unknown/component/UnknownContent.tsx +++ b/apps/web/src/features/block-unknown/component/UnknownContent.tsx @@ -1,19 +1,41 @@ +import { LoadingSpinner } from '@core/component/LoadingSpinner'; import ShareFat from '@icon/share.svg'; import DownloadSimple from '@phosphor/download-simple.svg'; import { Button } from '@ui'; +import { Show } from 'solid-js'; export function UnknownContent(props: { fileName: string; + /** Set while a legacy Office file is being upgraded, e.g. "PowerPoint presentation". */ + convertingTo?: string; onShare: () => void; onDownload: () => void; }) { return (
-
- No preview available for{' '} - {props.fileName} -
+ + No preview available for{' '} + {props.fileName} +
+ } + > + {(convertingTo) => ( +
+ +
+ Converting {props.fileName}{' '} + to a {convertingTo()}… +
+
+ )} +