diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json index f5680947d89..4e731e7c62c 100644 --- a/.github/workspace-dep-closures.json +++ b/.github/workspace-dep-closures.json @@ -4877,6 +4877,7 @@ "crates/macro_env_var", "crates/macro_event_broker", "crates/macro_event_topics", + "crates/macro_queues", "crates/macro_service_urls", "crates/macro_sync_service_jwt", "crates/macro_tower_layers", diff --git a/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json b/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json new file mode 100644 index 00000000000..1b5d58ed199 --- /dev/null +++ b/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \"Document\"\n SET \"fileType\" = $3,\n \"updatedAt\" = NOW()\n WHERE id = $1 AND LOWER(\"fileType\") = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c" +} diff --git a/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json b/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json new file mode 100644 index 00000000000..585499ecb11 --- /dev/null +++ b/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO \"DocumentBom\" (\"documentId\", \"createdAt\", \"updatedAt\")\n VALUES ($1, NOW(), NOW())\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30" +} diff --git a/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json b/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json new file mode 100644 index 00000000000..c8b812d4563 --- /dev/null +++ b/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM \"DocumentInstance\" WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c" +} diff --git a/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json b/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json new file mode 100644 index 00000000000..ee430a934f7 --- /dev/null +++ b/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM \"DocumentBom\" WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c" +} diff --git a/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json b/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json new file mode 100644 index 00000000000..208428c2618 --- /dev/null +++ b/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO \"DocumentInstance\" (\"documentId\", \"sha\", \"createdAt\", \"updatedAt\")\n VALUES ($1, $2, NOW(), NOW())\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3" +} diff --git a/Cargo.lock b/Cargo.lock index c2f4f11447e..4e7a4336e5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4802,6 +4802,7 @@ dependencies = [ "aws-sdk-sqs", "axum", "database_env_vars", + "docx_engine", "doppler-config", "futures", "http-body-util", @@ -4820,6 +4821,7 @@ dependencies = [ "macro_uuid", "model", "nix 0.29.0", + "pptx_engine", "remote_env_var", "rs-libreoffice-bindings", "s3_client", @@ -4830,6 +4832,7 @@ dependencies = [ "sqlx", "sqs_client", "sqs_worker", + "tempfile", "tokio", "tower 0.5.3", "tower-http", @@ -6396,12 +6399,17 @@ dependencies = [ "macro_aws_config", "macro_entrypoint", "macro_env_var", + "macro_queues", "macro_service_urls", + "macro_uuid", "model", + "model_owner", "s3_key", "serde", "serde_json", + "sha2 0.10.9", "sqlx", + "sqs_client", "sync_service_client", "tokio", "tracing", diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md index d494582b0dd..edd7863d15b 100644 --- a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md @@ -49,3 +49,5 @@ Notices: - U.S. federal works (SEC, Census, BLS, BEA, IRS, SBA, GSA) are in the public domain; the FRED export carries BLS series data. Templates whose terms forbid redistribution (Microsoft Create, Vertex42, Breaking Into Wall Street, Damodaran, SBDC) were used only for local testing and are not committed. Add a file by copying it here unmodified and adding its `file`, `title`, `source` and `license` to the manifest; set `skipCalculation` with a reason only when the engine cannot finish it. + +`upgraded-from-xls.xlsx` is a legacy Excel 97 workbook (`services/convert_service/fixtures/legacy/budget.xls`) converted to `.xlsx` by LibreOffice with the `Calc MS Excel 2007 XML` filter, exactly as the convert service upgrades uploaded `.xls` files. `../xlsx-upgraded-legacy.test.ts` checks that it imports into a native spreadsheet with its sheets, formulas, merges and number formats. diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx new file mode 100644 index 00000000000..775d52de0ae Binary files /dev/null and b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx differ diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts new file mode 100644 index 00000000000..13f4282bac4 --- /dev/null +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts @@ -0,0 +1,122 @@ +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { initSync } from '@ironcalc/wasm'; +import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation'; +import { + importSpreadsheetSheets, + readSpreadsheetWorkbook, +} from '@macro-inc/spreadsheet/workbook-document'; +import ExcelJS from 'exceljs'; +import { LoroDoc } from 'loro-crdt'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { decodeXlsx, encodeXlsx } from './xlsx-codec'; + +/** + * An uploaded legacy `.xls` is upgraded to `.xlsx` by LibreOffice on the + * server; this is that output for a two-sheet budget. It must import into a + * native spreadsheet like any Excel-written workbook. + */ +const upgraded = () => + new Uint8Array( + readFileSync( + createRequire(import.meta.url).resolve( + './xlsx-fixtures/upgraded-from-xls.xlsx' + ) + ) + ); + +beforeAll(() => + initSync({ + module: readFileSync( + createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm') + ), + }) +); + +describe('workbook upgraded from a legacy .xls', () => { + it('imports both sheets with their cells and formulas', async () => { + const imported = await decodeXlsx(upgraded()); + + expect(imported.sheets.map((sheet) => sheet.name)).toEqual([ + 'Budget', + 'Notes', + ]); + const [budget, notes] = imported.sheets; + expect(budget.cells.A1.value).toBe('Month'); + // Stored as text in the .xls, so it stays text rather than becoming a date. + expect(budget.cells.A2.value).toBe("'2026-01"); + expect(budget.cells.B2.value).toBe('1050'); + expect(budget.cells.D2.value).toBe('=B2-C2'); + expect(budget.cells.D13.value).toBe('=B13-C13'); + expect(budget.cells.A15.value).toBe('Total'); + expect(budget.cells.B15.value).toBe('=SUM(B2:B13)'); + expect(budget.cells.D15.value).toBe('=SUM(D2:D13)'); + expect(budget.cells.F1.value).toBe('Merged header'); + expect(notes.cells.B1.value).toBe('Finance'); + expect(notes.cells.B2.value).toBe('=Budget!D15/Budget!B15'); + }); + + it('keeps header styles, number formats and the merged header', async () => { + const imported = await decodeXlsx(upgraded()); + const budget = imported.sheets[0]; + + expect(budget.cells.A1).toMatchObject({ + bold: true, + fillColor: '#DDEBF7', + }); + // `#,##0.00` imports as Macro's thousands-separated number format. + expect(budget.cells.B15).toMatchObject({ format: 'number', decimals: 2 }); + expect(budget.metadata?.merges).toEqual(['F1:H1']); + expect(imported.warnings).toEqual([ + 'Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.', + ]); + }); + + it('calculates the same results after native save and reopen', async () => { + const imported = await decodeXlsx(upgraded()); + const doc = new LoroDoc(); + const reopened = new LoroDoc(); + const engine = createInitializedSpreadsheetCalculator(); + try { + importSpreadsheetSheets(doc, imported.sheets, true); + reopened.import(doc.export({ mode: 'snapshot' })); + const sheets = readSpreadsheetWorkbook(reopened); + const results = engine.calculateWorkbook( + sheets.map((sheet) => ({ ...sheet, rowCount: sheet.layout.rowCount })), + { includeTypes: true } + ); + + const budget = results[sheets[0].id]; + // Revenue 1000 + 50i and cost 600 + 20i for months 1..12. + expect(budget.B15.number).toBe(15_900); + expect(budget.C15.number).toBe(8_760); + expect(budget.D15.number).toBe(7_140); + expect(budget.D2.number).toBe(430); + expect(budget.B15.display).toBe('15,900.00'); + expect(results[sheets[1].id].B2.number).toBeCloseTo(7_140 / 15_900, 12); + for (const values of Object.values(results)) { + expect( + Object.entries(values).filter(([, value]) => value.error) + ).toEqual([]); + } + } finally { + engine.dispose(); + } + }); + + it('exports back to an .xlsx Excel reads with the same formulas', async () => { + const imported = await decodeXlsx(upgraded()); + + const exported = await encodeXlsx({ sheets: imported.sheets }); + + const external = new ExcelJS.Workbook(); + await external.xlsx.load(exported.bytes.slice().buffer); + const budget = external.getWorksheet('Budget')!; + expect(budget.getCell('B15').formula).toBe('SUM(B2:B13)'); + expect(budget.getCell('D2').formula).toBe('B2-C2'); + expect(budget.getCell('F1').isMerged).toBe(true); + expect(external.getWorksheet('Notes')!.getCell('B2').formula).toBe( + 'Budget!D15/Budget!B15' + ); + }); +}); diff --git a/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx b/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx index 42cdf952b13..99c0998a08c 100644 --- a/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx +++ b/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx @@ -529,9 +529,16 @@ export function SpreadsheetEditor(props: { async function importFile(file: File | undefined) { if (!file || !editable()) return; + if (/\.xls$/i.test(file.name)) { + // Converting .xls to .xlsx needs the server; uploads are upgraded. + actions.setNotice( + 'Legacy .xls workbooks are converted when uploaded. Upload the file, then choose Edit in Macro.' + ); + return; + } if (!/\.(csv|xlsx)$/i.test(file.name)) { actions.setNotice( - 'Choose a .csv or .xlsx file. Legacy .xls and macro-enabled workbooks are not supported.' + 'Choose a .csv or .xlsx file. Macro-enabled workbooks are not supported.' ); return; } diff --git a/apps/web/src/features/block-unknown/component/Block.tsx b/apps/web/src/features/block-unknown/component/Block.tsx index dc7355e4e43..897d74ae1fe 100644 --- a/apps/web/src/features/block-unknown/component/Block.tsx +++ b/apps/web/src/features/block-unknown/component/Block.tsx @@ -1,8 +1,10 @@ import { FileSidePanelSections, SidePanel } from '@components/app/side-panel'; +import { useSplitLayout } from '@components/app/split-layout/layout'; import { useBlockId } from '@core/block'; import { DocumentBlockContainer } from '@core/component/DocumentBlockContainer'; import { toast } from '@core/component/Toast/Toast'; import { useShareModal } from '@core/component/TopBar/shareModal'; +import { fileTypeToBlockName } from '@core/constant/allBlocks'; import { blockMetadataSignal } from '@core/signal/load'; import { useGetPermissions } from '@core/signal/permissions'; import { @@ -10,11 +12,26 @@ import { useBlockDocumentName, } from '@core/util/currentBlockDocumentName'; import { downloadFile } from '@filesystem/download'; +import { waitForDocumentContentReady } from '@queries/storage/document-location'; +import { fetchDocumentMetadata } from '@queries/storage/document-metadata'; import { createCallback } from '@solid-primitives/rootless'; -import { lazy, Show, Suspense } from 'solid-js'; +import { + createSignal, + lazy, + onCleanup, + onMount, + Show, + Suspense, +} from 'solid-js'; import { SpreadsheetSkeleton } from '../../block-spreadsheet/components/SpreadsheetSkeleton'; import { isUploadedWorkbook } from '../../block-spreadsheet/core/uploaded-workbook'; import { useSpreadsheetAccess } from '../../block-spreadsheet/primitives/use-spreadsheet-access'; +import { + legacyOfficeUpgradeTarget, + pollForLegacyUpgrade, + shouldAwaitLegacyUpgrade, + upgradedOfficeLabel, +} from '../legacy-office'; import { useGetFileBlob } from '../signal/blockData'; import { TopBar } from './TopBar'; import { UnknownContent } from './UnknownContent'; @@ -50,6 +67,7 @@ function BlockUnknownContent() { owner: blockMetadataSignal()?.owner, })); const getBlob = useGetFileBlob(); + const convertingTo = useLegacyOfficeUpgrade(blockId); const downloadDocument = createCallback(async () => { try { @@ -74,6 +92,7 @@ function BlockUnknownContent() { fallback={ void downloadDocument()} /> @@ -88,3 +107,56 @@ function BlockUnknownContent() { ); } + +/** + * Waits for a just-uploaded .doc/.ppt/.xls to be upgraded to OpenXML, then + * reopens it in the block for its new type. Returns the label of the format + * being converted to while waiting. + */ +function useLegacyOfficeUpgrade(documentId: string) { + const { replaceSplit } = useSplitLayout(); + const initial = blockMetadataSignal.get(); + const target = legacyOfficeUpgradeTarget(initial?.fileType); + const [waiting, setWaiting] = createSignal( + !!initial && !!target && shouldAwaitLegacyUpgrade(initial, Date.now()) + ); + + onMount(() => { + if (!target || !waiting()) return; + const controller = new AbortController(); + onCleanup(() => controller.abort()); + + void (async () => { + const upgraded = await pollForLegacyUpgrade({ + target, + fetchMetadata: () => fetchDocumentMetadata(documentId), + fileTypeOf: (metadata) => metadata.fileType, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + signal: controller.signal, + }); + if (controller.signal.aborted) return; + if (!upgraded) { + setWaiting(false); + return; + } + // Word content becomes readable once the DOCX pipeline processes it. + if (target === 'docx') { + await waitForDocumentContentReady({ + documentId, + timeoutMs: 60_000, + }).catch(() => undefined); + if (controller.signal.aborted) return; + } + const blockName = fileTypeToBlockName(upgraded.fileType); + if (blockName === 'unknown') { + // Excel workbooks stay in this block, which previews them. + blockMetadataSignal.set(upgraded); + setWaiting(false); + return; + } + replaceSplit({ content: { type: blockName, id: documentId } }); + })(); + }); + + return () => (waiting() && target ? upgradedOfficeLabel(target) : undefined); +} diff --git a/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx b/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx new file mode 100644 index 00000000000..25d6483c16c --- /dev/null +++ b/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx @@ -0,0 +1,38 @@ +import { cleanup, render, screen } from '@solidjs/testing-library'; +import { afterEach, expect, it, vi } from 'vitest'; +import { UnknownContent } from './UnknownContent'; + +afterEach(cleanup); + +it('offers the file when there is no preview', () => { + render(() => ( + + )); + + expect(screen.getByText(/No preview available for/)).toBeTruthy(); + expect(screen.queryByTestId('legacy-office-converting')).toBeNull(); + expect(screen.getByText('Download')).toBeTruthy(); +}); + +it('says a legacy Office file is being converted and still offers the original', () => { + const onDownload = vi.fn(); + render(() => ( + + )); + + const status = screen.getByTestId('legacy-office-converting'); + expect(status.textContent).toContain('Board Update.ppt'); + expect(status.textContent).toContain('to a PowerPoint presentation'); + expect(screen.queryByText(/No preview available/)).toBeNull(); + screen.getByText('Download').click(); + expect(onDownload).toHaveBeenCalledOnce(); +}); diff --git a/apps/web/src/features/block-unknown/component/UnknownContent.tsx b/apps/web/src/features/block-unknown/component/UnknownContent.tsx index 98c9ee2c75e..64f44dfae2c 100644 --- a/apps/web/src/features/block-unknown/component/UnknownContent.tsx +++ b/apps/web/src/features/block-unknown/component/UnknownContent.tsx @@ -1,19 +1,41 @@ +import { LoadingSpinner } from '@core/component/LoadingSpinner'; import ShareFat from '@icon/share.svg'; import DownloadSimple from '@phosphor/download-simple.svg'; import { Button } from '@ui'; +import { Show } from 'solid-js'; export function UnknownContent(props: { fileName: string; + /** Set while a legacy Office file is being upgraded, e.g. "PowerPoint presentation". */ + convertingTo?: string; onShare: () => void; onDownload: () => void; }) { return (
-
- No preview available for{' '} - {props.fileName} -
+ + No preview available for{' '} + {props.fileName} +
+ } + > + {(convertingTo) => ( +
+ +
+ Converting {props.fileName}{' '} + to a {convertingTo()}… +
+
+ )} +