diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json index f5680947d89..4e731e7c62c 100644 --- a/.github/workspace-dep-closures.json +++ b/.github/workspace-dep-closures.json @@ -4877,6 +4877,7 @@ "crates/macro_env_var", "crates/macro_event_broker", "crates/macro_event_topics", + "crates/macro_queues", "crates/macro_service_urls", "crates/macro_sync_service_jwt", "crates/macro_tower_layers", diff --git a/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json b/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json new file mode 100644 index 00000000000..1b5d58ed199 --- /dev/null +++ b/.sqlx/query-1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \"Document\"\n SET \"fileType\" = $3,\n \"updatedAt\" = NOW()\n WHERE id = $1 AND LOWER(\"fileType\") = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1f21761b4bc48d82468fa7e8cebbd5e309aab299895ab0e4a91c01a870d9109c" +} diff --git a/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json b/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json new file mode 100644 index 00000000000..585499ecb11 --- /dev/null +++ b/.sqlx/query-3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO \"DocumentBom\" (\"documentId\", \"createdAt\", \"updatedAt\")\n VALUES ($1, NOW(), NOW())\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3f6fad37bbed718a3124a1946c642b63e816133ecea6ce72c8c9507277617f30" +} diff --git a/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json b/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json new file mode 100644 index 00000000000..c8b812d4563 --- /dev/null +++ b/.sqlx/query-92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM \"DocumentInstance\" WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "92d93a6bc6a48cce58a03908a840395eb184c9eb259894f689390b13e5647a6c" +} diff --git a/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json b/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json new file mode 100644 index 00000000000..ee430a934f7 --- /dev/null +++ b/.sqlx/query-dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM \"DocumentBom\" WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "dadea3eca110a122079f319ea9d944cdab75327b002a16e85cf9da6fe8f18c8c" +} diff --git a/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json b/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json new file mode 100644 index 00000000000..208428c2618 --- /dev/null +++ b/.sqlx/query-e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO \"DocumentInstance\" (\"documentId\", \"sha\", \"createdAt\", \"updatedAt\")\n VALUES ($1, $2, NOW(), NOW())\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e53dd219e78de6f92272433200e09f2d94ad20d9f6e193e1e92655d335a128d3" +} diff --git a/Cargo.lock b/Cargo.lock index c2f4f11447e..4e7a4336e5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4802,6 +4802,7 @@ dependencies = [ "aws-sdk-sqs", "axum", "database_env_vars", + "docx_engine", "doppler-config", "futures", "http-body-util", @@ -4820,6 +4821,7 @@ dependencies = [ "macro_uuid", "model", "nix 0.29.0", + "pptx_engine", "remote_env_var", "rs-libreoffice-bindings", "s3_client", @@ -4830,6 +4832,7 @@ dependencies = [ "sqlx", "sqs_client", "sqs_worker", + "tempfile", "tokio", "tower 0.5.3", "tower-http", @@ -6396,12 +6399,17 @@ dependencies = [ "macro_aws_config", "macro_entrypoint", "macro_env_var", + "macro_queues", "macro_service_urls", + "macro_uuid", "model", + "model_owner", "s3_key", "serde", "serde_json", + "sha2 0.10.9", "sqlx", + "sqs_client", "sync_service_client", "tokio", "tracing", diff --git a/apps/web/scripts/agent-fold.test.ts b/apps/web/scripts/agent-fold.test.ts index 36fa371e754..e85964654a9 100644 --- a/apps/web/scripts/agent-fold.test.ts +++ b/apps/web/scripts/agent-fold.test.ts @@ -18,7 +18,9 @@ beforeAll(async () => { module_or_path: readFileSync(new URL('agent_fold_bg.wasm', path)), }); Stream = wasm.FoldStream; -}, 300_000); + // CI compiles the release WASM from scratch on every run, which takes 3–6 + // minutes depending on the runner. +}, 600_000); it('decodes durable DTOs, reconciles overlap, and folds a batch like a stream', () => { const session = '00000000-0000-0000-0000-00000000000a'; diff --git a/apps/web/src/features/block-spreadsheet/browser-test/spreadsheet.browser.e2e.ts b/apps/web/src/features/block-spreadsheet/browser-test/spreadsheet.browser.e2e.ts index 87e7e572bad..6a17d7f5ae6 100644 --- a/apps/web/src/features/block-spreadsheet/browser-test/spreadsheet.browser.e2e.ts +++ b/apps/web/src/features/block-spreadsheet/browser-test/spreadsheet.browser.e2e.ts @@ -328,6 +328,71 @@ test('imports independent XLSX through the file picker and exports real formulas }); }); +test('imports a macro-enabled .xlsm without its macros and exports a macro-free .xlsx', async ({ + page, +}) => { + const chooser = page.waitForEvent('filechooser'); + await menu(page, 'Import and export', 'Import…'); + await (await chooser).setFiles( + fileURLToPath( + new URL('../core/xlsx-fixtures/poi-simple-macro.xlsm', import.meta.url) + ) + ); + const dialog = page.getByRole('dialog', { name: 'Import Excel workbook' }); + await expect(dialog).toBeVisible(); + await expect(dialog).toContainText('VBA macros are not imported.'); + await page.getByRole('radio', { name: /Replace workbook/ }).check(); + await page.getByRole('button', { name: 'Import workbook' }).click(); + await expect(page.getByRole('tab')).toHaveCount(3); + await expect(page.locator('[data-address="A1"]')).toHaveText( + 'This is a macro workbook' + ); + + const downloaded = page.waitForEvent('download'); + await menu(page, 'Import and export', 'Download as Excel (.xlsx)'); + const download = await downloaded; + expect(download.suggestedFilename()).toBe('Spreadsheet fixture.xlsx'); + const path = await download.path(); + if (!path) throw new Error('The workbook download was not saved.'); + const workbook = new ExcelJS.Workbook(); + await workbook.xlsx.readFile(path); + expect(workbook.worksheets.map((sheet) => sheet.name)).toEqual([ + 'Sheet1', + 'Sheet2', + 'Sheet3', + ]); + expect(workbook.getWorksheet('Sheet1')?.getCell('A1').value).toBe( + 'This is a macro workbook' + ); + const { unzipSync } = await import('fflate'); + const { readFileSync } = await import('node:fs'); + expect( + Object.keys(unzipSync(readFileSync(path))).filter((name) => + /vba/i.test(name) + ) + ).toEqual([]); +}); + +test('explains that legacy .xls workbooks convert when uploaded', async ({ + page, +}) => { + const chooser = page.waitForEvent('filechooser'); + await menu(page, 'Import and export', 'Import…'); + await (await chooser).setFiles({ + name: 'budget.xls', + mimeType: 'application/vnd.ms-excel', + buffer: Buffer.from([0xd0, 0xcf, 0x11, 0xe0]), + }); + await expect( + page.getByText( + 'Legacy .xls workbooks are converted when uploaded. Upload the file, then choose Edit in Macro.' + ) + ).toBeVisible(); + await expect( + page.getByRole('dialog', { name: 'Import Excel workbook' }) + ).toHaveCount(0); +}); + test('undo restores the original workbook after replacing it with an Excel import', async ({ page, }) => { diff --git a/apps/web/src/features/block-spreadsheet/core/csv-file.test.ts b/apps/web/src/features/block-spreadsheet/core/csv-file.test.ts index 8714b1424de..9253694e950 100644 --- a/apps/web/src/features/block-spreadsheet/core/csv-file.test.ts +++ b/apps/web/src/features/block-spreadsheet/core/csv-file.test.ts @@ -4,10 +4,11 @@ import { csvImportEdits } from './sheet-operations'; import { decodeCsv, isUploadedWorkbook } from './uploaded-workbook'; describe('uploaded Excel/CSV detection', () => { - it.each(['xlsx', 'XLSX', 'csv', 'CSV'])('recognizes %s', (type) => - expect(isUploadedWorkbook(type)).toBe(true) + it.each(['xlsx', 'XLSX', 'xlsm', 'XLSM', 'csv', 'CSV'])( + 'recognizes %s', + (type) => expect(isUploadedWorkbook(type)).toBe(true) ); - it.each(['xls', 'xlsm', 'pdf', 'spreadsheet', '', null, undefined])( + it.each(['xls', 'xltm', 'pdf', 'spreadsheet', '', null, undefined])( 'rejects %s', (type) => expect(isUploadedWorkbook(type)).toBe(false) ); diff --git a/apps/web/src/features/block-spreadsheet/core/uploaded-workbook.ts b/apps/web/src/features/block-spreadsheet/core/uploaded-workbook.ts index df0551bb3d6..bbee229b5d2 100644 --- a/apps/web/src/features/block-spreadsheet/core/uploaded-workbook.ts +++ b/apps/web/src/features/block-spreadsheet/core/uploaded-workbook.ts @@ -6,7 +6,7 @@ import { import type { WorkbookFileData } from './workbook-file-types'; export function isUploadedWorkbook(fileType: string | null | undefined) { - return /^(xlsx|csv)$/i.test(fileType ?? ''); + return /^(xlsx|xlsm|csv)$/i.test(fileType ?? ''); } /** CSV carries values, not executable formulas or cell types. Keep identifiers diff --git a/apps/web/src/features/block-spreadsheet/core/xlsm-import.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsm-import.test.ts new file mode 100644 index 00000000000..f40600c7f43 --- /dev/null +++ b/apps/web/src/features/block-spreadsheet/core/xlsm-import.test.ts @@ -0,0 +1,220 @@ +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { initSync } from '@ironcalc/wasm'; +import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation'; +import { + importSpreadsheetSheets, + readSpreadsheetWorkbook, +} from '@macro-inc/spreadsheet/workbook-document'; +import ExcelJS from 'exceljs'; +import { strFromU8, strToU8, unzipSync, zipSync } from 'fflate'; +import { LoroDoc } from 'loro-crdt'; +import { beforeAll, describe, expect, it } from 'vitest'; +import type { WorkbookFileSheet } from './workbook-file-types'; +import { inspectXlsxArchive } from './xlsx-archive'; +import { decodeXlsx, encodeXlsx } from './xlsx-codec'; + +const VBA_WARNING = + 'VBA macros are not imported. Cells, formulas and formatting are; the macros do not run in Macro.'; +const MACRO_SHEET_WARNING = 'Excel 4.0 macro sheets are not imported.'; +const CONTROLS_WARNING = 'Form and ActiveX controls are not imported.'; + +const fixture = (name: string) => + new Uint8Array( + readFileSync( + createRequire(import.meta.url).resolve(`./xlsx-fixtures/${name}`) + ) + ); + +beforeAll(() => + initSync({ + module: readFileSync( + createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm') + ), + }) +); + +function budgetSheet(): WorkbookFileSheet { + return { + name: 'Budget', + cells: { + A1: { value: 'Item', bold: true }, + B1: { value: 'Amount', bold: true }, + A2: { value: 'Rent' }, + B2: { value: '1200' }, + A3: { value: 'Payroll' }, + B3: { value: '4800' }, + A4: { value: 'Total' }, + B4: { value: '=SUM(B2:B3)' }, + }, + rowCount: 100, + columnWidths: {}, + }; +} + +/** + * A Macro-written workbook turned into the package Excel saves as .xlsm: the + * macro-enabled main content type, a VBA project and its signature, plus any + * `extra` parts. + */ +async function macroEnabled(extra: Record = {}) { + const { bytes } = await encodeXlsx({ sheets: [budgetSheet()] }); + const entries = unzipSync(bytes); + entries['[Content_Types].xml'] = strToU8( + strFromU8(entries['[Content_Types].xml']) + .replace( + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml', + 'application/vnd.ms-excel.sheet.macroEnabled.main+xml' + ) + .replace( + '', + '' + ) + ); + entries['xl/_rels/workbook.xml.rels'] = strToU8( + strFromU8(entries['xl/_rels/workbook.xml.rels']).replace( + '', + '' + ) + ); + // An OLE compound file header; the importer never reads the VBA project. + entries['xl/vbaProject.bin'] = new Uint8Array([ + 0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, + ]); + entries['xl/vbaProjectSignature.bin'] = new Uint8Array([1, 2, 3]); + Object.assign(entries, extra); + return zipSync(entries); +} + +async function exportedParts(sheets: WorkbookFileSheet[]) { + const { bytes } = await encodeXlsx({ sheets }); + return { bytes, names: Object.keys(unzipSync(bytes)) }; +} + +describe('macro-enabled workbook import', () => { + it('imports a macro-enabled workbook with its formulas and a note about the macros', async () => { + const imported = await decodeXlsx(await macroEnabled()); + + expect(imported.sheets.map((sheet) => sheet.name)).toEqual(['Budget']); + expect(imported.sheets[0].cells.A4.value).toBe('Total'); + expect(imported.sheets[0].cells.B4.value).toBe('=SUM(B2:B3)'); + expect(imported.sheets[0].cells.A1.bold).toBe(true); + expect(imported.warnings).toContain(VBA_WARNING); + expect(imported.warnings).not.toContain(MACRO_SHEET_WARNING); + expect(imported.warnings).not.toContain(CONTROLS_WARNING); + }); + + it('notes Excel 4.0 macro sheets and form controls separately', async () => { + const imported = await decodeXlsx( + await macroEnabled({ + 'xl/macrosheets/sheet2.xml': strToU8( + '' + ), + 'xl/activeX/activeX1.xml': strToU8(''), + 'xl/ctrlProps/ctrlProp1.xml': strToU8(''), + }) + ); + + expect(imported.warnings).toEqual( + expect.arrayContaining([ + VBA_WARNING, + MACRO_SHEET_WARNING, + CONTROLS_WARNING, + ]) + ); + expect(imported.sheets[0].cells.B4.value).toBe('=SUM(B2:B3)'); + }); + + it('calculates an imported macro-enabled workbook after native save and reopen', async () => { + const imported = await decodeXlsx(await macroEnabled()); + const doc = new LoroDoc(); + const reopened = new LoroDoc(); + const engine = createInitializedSpreadsheetCalculator(); + try { + importSpreadsheetSheets(doc, imported.sheets, true); + reopened.import(doc.export({ mode: 'snapshot' })); + const sheets = readSpreadsheetWorkbook(reopened); + const results = engine.calculateWorkbook( + sheets.map((sheet) => ({ ...sheet, rowCount: sheet.layout.rowCount })), + { includeTypes: true } + ); + expect(results[sheets[0].id].B4.number).toBe(6_000); + } finally { + engine.dispose(); + } + }); + + it('exports an imported macro-enabled workbook as a macro-free .xlsx', async () => { + const imported = await decodeXlsx(await macroEnabled()); + + const { bytes, names } = await exportedParts(imported.sheets); + + expect(names.some((name) => /vba|macrosheets|activeX/i.test(name))).toBe( + false + ); + const contentTypes = strFromU8(unzipSync(bytes)['[Content_Types].xml']); + expect(contentTypes).toContain( + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml' + ); + expect(contentTypes).not.toContain('macroEnabled'); + const external = new ExcelJS.Workbook(); + await external.xlsx.load(bytes.slice().buffer); + expect(external.getWorksheet('Budget')!.getCell('B4').formula).toBe( + 'SUM(B2:B3)' + ); + }); + + it("still refuses macros in Macro's own export check", async () => { + expect(() => inspectXlsxArchive(fixture('poi-simple-macro.xlsm'))).toThrow( + 'must not contain macros' + ); + expect(() => + inspectXlsxArchive( + zipSync({ + ...unzipSync(fixture('poi-simple-macro.xlsm')), + 'xl/vbaProject.bin': new Uint8Array(), + }) + ) + ).toThrow('must not contain macros'); + }); +}); + +describe('Excel-written .xlsm files', () => { + it('imports a simple macro workbook', async () => { + const imported = await decodeXlsx(fixture('poi-simple-macro.xlsm')); + + expect(imported.sheets.map((sheet) => sheet.name)).toEqual([ + 'Sheet1', + 'Sheet2', + 'Sheet3', + ]); + expect(imported.sheets[0].cells.A1.value).toBe('This is a macro workbook'); + expect(imported.warnings).toContain(VBA_WARNING); + + const { names } = await exportedParts(imported.sheets); + expect(names).not.toContain('xl/vbaProject.bin'); + }); + + it('imports a macro workbook with comments, embedded objects and Cyrillic sheet names', async () => { + const imported = await decodeXlsx( + fixture('poi-excel-with-attachments.xlsm') + ); + + expect(imported.sheets.map((sheet) => sheet.name)).toEqual([ + 'Страница 1', + 'Page 1', + 'Бетўўў 1', + ]); + const text = imported.sheets.flatMap((sheet) => + Object.values(sheet.cells).map((cell) => cell.value) + ); + expect(text).toEqual( + expect.arrayContaining(['Русский вариант', 'English', 'Ўзбек']) + ); + expect(imported.warnings).toContain(VBA_WARNING); + + const { bytes, names } = await exportedParts(imported.sheets); + expect(names.some((name) => /vbaProject/i.test(name))).toBe(false); + expect(() => inspectXlsxArchive(bytes)).not.toThrow(); + }); +}); diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-archive.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-archive.ts index 77d7d58029f..5e068fade9c 100644 --- a/apps/web/src/features/block-spreadsheet/core/xlsx-archive.ts +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-archive.ts @@ -29,7 +29,10 @@ type ArchiveEntry = { }; /** Validate the ZIP directory: bounded, unencrypted, no path aliases. */ -function archiveEntries(bytes: Uint8Array): { +function archiveEntries( + bytes: Uint8Array, + { allowMacros }: { allowMacros: boolean } +): { entries: ArchiveEntry[]; centralStart: number; } { @@ -115,10 +118,10 @@ function archiveEntries(bytes: Uint8Array): { !names.has('xl/workbook.xml') ) throw invalid(); - if ([...names].some((name) => /vbaProject\.bin$/i.test(name))) - throw new Error( - 'Macro-enabled workbooks are not supported. Save a macro-free .xlsx copy first.' - ); + // Imports drop macros with a warning; Macro's own exports must never + // contain them. + if (!allowMacros && [...names].some(isMacroPart)) + throw new Error('Workbooks written by Macro must not contain macros.'); return { entries, centralStart }; } @@ -159,7 +162,9 @@ function inflateEntry( /** Validate ZIP metadata and inflate every entry into fixed buffers. */ export function inspectXlsxArchive(bytes: Uint8Array) { - const { entries, centralStart } = archiveEntries(bytes); + const { entries, centralStart } = archiveEntries(bytes, { + allowMacros: false, + }); const files: Record = Object.create(null); for (const entry of entries) files[entry.name] = inflateEntry(bytes, entry, centralStart); @@ -174,7 +179,9 @@ export type XlsxArchive = { /** Validate the directory up front, then inflate entries only when read. */ export function openXlsxArchive(bytes: Uint8Array): XlsxArchive { - const { entries, centralStart } = archiveEntries(bytes); + const { entries, centralStart } = archiveEntries(bytes, { + allowMacros: true, + }); const byName = new Map(entries.map((entry) => [entry.name, entry])); return { names: [...byName.keys()], @@ -185,6 +192,14 @@ export function openXlsxArchive(bytes: Uint8Array): XlsxArchive { }; } +/** VBA projects, their signatures, and Excel 4.0 macro sheets. */ +function isMacroPart(name: string) { + return ( + /(^|\/)vbaProject(Signature(Agile|V3)?)?\.bin$/i.test(name) || + /^xl\/macrosheets\//i.test(name) + ); +} + /** Warnings for package parts Macro does not import, from entry names alone. */ export function xlsxFeatureWarnings(names: string[]): string[] { const warnings = new Set(); @@ -197,6 +212,14 @@ export function xlsxFeatureWarnings(names: string[]): string[] { warnings.add( 'Excel tables are imported as ordinary cells; table formulas are converted to cell ranges.' ); + if (/(^|\/)vbaProject\.bin$/i.test(name)) + warnings.add( + 'VBA macros are not imported. Cells, formulas and formatting are; the macros do not run in Macro.' + ); + if (/^xl\/macrosheets\//i.test(name)) + warnings.add('Excel 4.0 macro sheets are not imported.'); + if (/^xl\/(activeX|ctrlProps)\//i.test(name)) + warnings.add('Form and ActiveX controls are not imported.'); } return [...warnings]; } diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-codec.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-codec.test.ts index ac5dbe0d036..71d03710493 100644 --- a/apps/web/src/features/block-spreadsheet/core/xlsx-codec.test.ts +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-codec.test.ts @@ -564,7 +564,7 @@ describe('Excel workbook files', () => { expect(result.rowCount).toBe(5000); }); - it('rejects invalid ZIP, encryption, macros, corrupt data and forged expansion sizes', async () => { + it('rejects invalid ZIP, encryption, exported macros, corrupt data and forged expansion sizes', async () => { await expect( decodeXlsx(new Uint8Array(XLSX_MAX_BYTES + 1)) ).rejects.toThrow('50 MB'); @@ -574,7 +574,10 @@ describe('Excel workbook files', () => { const original = (await encodeXlsx({ sheets: [simpleSheet()] })).bytes; const entries = unzipSync(original); entries['xl/vbaProject.bin'] = new Uint8Array([1]); - await expect(decodeXlsx(zipSync(entries))).rejects.toThrow('Macro-enabled'); + // Imports drop macros (see xlsm-import.test.ts); exports may never carry them. + expect(() => inspectXlsxArchive(zipSync(entries))).toThrow( + 'must not contain macros' + ); const central = (bytes: Uint8Array) => { const view = new DataView( bytes.buffer, diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md index d494582b0dd..e763c1f2a90 100644 --- a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/README.md @@ -49,3 +49,7 @@ Notices: - U.S. federal works (SEC, Census, BLS, BEA, IRS, SBA, GSA) are in the public domain; the FRED export carries BLS series data. Templates whose terms forbid redistribution (Microsoft Create, Vertex42, Breaking Into Wall Street, Damodaran, SBDC) were used only for local testing and are not committed. Add a file by copying it here unmodified and adding its `file`, `title`, `source` and `license` to the manifest; set `skipCalculation` with a reason only when the engine cannot finish it. + +`upgraded-from-xls.xlsx` is a legacy Excel 97 workbook (`services/convert_service/fixtures/legacy/budget.xls`) converted to `.xlsx` by LibreOffice with the `Calc MS Excel 2007 XML` filter, exactly as the convert service upgrades uploaded `.xls` files. `../xlsx-upgraded-legacy.test.ts` checks that it imports into a native spreadsheet with its sheets, formulas, merges and number formats. + +`poi-simple-macro.xlsm` and `poi-excel-with-attachments.xlsm` are Excel-written macro-enabled workbooks with real VBA projects, from the Apache POI test data (`test-data/spreadsheet/SimpleMacro.xlsm` and `ExcelWithAttachments.xlsm` at https://github.com/apache/poi, Apache-2.0). The second also has comments, embedded Word and PowerPoint objects and Cyrillic sheet names. `../xlsm-import.test.ts` checks that both import with their macros dropped and export as macro-free `.xlsx`. diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/poi-excel-with-attachments.xlsm b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/poi-excel-with-attachments.xlsm new file mode 100644 index 00000000000..cb3e564428a Binary files /dev/null and b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/poi-excel-with-attachments.xlsm differ diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/poi-simple-macro.xlsm b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/poi-simple-macro.xlsm new file mode 100644 index 00000000000..ec64525bf8f Binary files /dev/null and b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/poi-simple-macro.xlsm differ diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx new file mode 100644 index 00000000000..775d52de0ae Binary files /dev/null and b/apps/web/src/features/block-spreadsheet/core/xlsx-fixtures/upgraded-from-xls.xlsx differ diff --git a/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts b/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts new file mode 100644 index 00000000000..13f4282bac4 --- /dev/null +++ b/apps/web/src/features/block-spreadsheet/core/xlsx-upgraded-legacy.test.ts @@ -0,0 +1,122 @@ +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { initSync } from '@ironcalc/wasm'; +import { createInitializedSpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation'; +import { + importSpreadsheetSheets, + readSpreadsheetWorkbook, +} from '@macro-inc/spreadsheet/workbook-document'; +import ExcelJS from 'exceljs'; +import { LoroDoc } from 'loro-crdt'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { decodeXlsx, encodeXlsx } from './xlsx-codec'; + +/** + * An uploaded legacy `.xls` is upgraded to `.xlsx` by LibreOffice on the + * server; this is that output for a two-sheet budget. It must import into a + * native spreadsheet like any Excel-written workbook. + */ +const upgraded = () => + new Uint8Array( + readFileSync( + createRequire(import.meta.url).resolve( + './xlsx-fixtures/upgraded-from-xls.xlsx' + ) + ) + ); + +beforeAll(() => + initSync({ + module: readFileSync( + createRequire(import.meta.url).resolve('@ironcalc/wasm/wasm_bg.wasm') + ), + }) +); + +describe('workbook upgraded from a legacy .xls', () => { + it('imports both sheets with their cells and formulas', async () => { + const imported = await decodeXlsx(upgraded()); + + expect(imported.sheets.map((sheet) => sheet.name)).toEqual([ + 'Budget', + 'Notes', + ]); + const [budget, notes] = imported.sheets; + expect(budget.cells.A1.value).toBe('Month'); + // Stored as text in the .xls, so it stays text rather than becoming a date. + expect(budget.cells.A2.value).toBe("'2026-01"); + expect(budget.cells.B2.value).toBe('1050'); + expect(budget.cells.D2.value).toBe('=B2-C2'); + expect(budget.cells.D13.value).toBe('=B13-C13'); + expect(budget.cells.A15.value).toBe('Total'); + expect(budget.cells.B15.value).toBe('=SUM(B2:B13)'); + expect(budget.cells.D15.value).toBe('=SUM(D2:D13)'); + expect(budget.cells.F1.value).toBe('Merged header'); + expect(notes.cells.B1.value).toBe('Finance'); + expect(notes.cells.B2.value).toBe('=Budget!D15/Budget!B15'); + }); + + it('keeps header styles, number formats and the merged header', async () => { + const imported = await decodeXlsx(upgraded()); + const budget = imported.sheets[0]; + + expect(budget.cells.A1).toMatchObject({ + bold: true, + fillColor: '#DDEBF7', + }); + // `#,##0.00` imports as Macro's thousands-separated number format. + expect(budget.cells.B15).toMatchObject({ format: 'number', decimals: 2 }); + expect(budget.metadata?.merges).toEqual(['F1:H1']); + expect(imported.warnings).toEqual([ + 'Merged ranges are shown as individual cells in Macro and restored on Excel export when their covered cells remain empty.', + ]); + }); + + it('calculates the same results after native save and reopen', async () => { + const imported = await decodeXlsx(upgraded()); + const doc = new LoroDoc(); + const reopened = new LoroDoc(); + const engine = createInitializedSpreadsheetCalculator(); + try { + importSpreadsheetSheets(doc, imported.sheets, true); + reopened.import(doc.export({ mode: 'snapshot' })); + const sheets = readSpreadsheetWorkbook(reopened); + const results = engine.calculateWorkbook( + sheets.map((sheet) => ({ ...sheet, rowCount: sheet.layout.rowCount })), + { includeTypes: true } + ); + + const budget = results[sheets[0].id]; + // Revenue 1000 + 50i and cost 600 + 20i for months 1..12. + expect(budget.B15.number).toBe(15_900); + expect(budget.C15.number).toBe(8_760); + expect(budget.D15.number).toBe(7_140); + expect(budget.D2.number).toBe(430); + expect(budget.B15.display).toBe('15,900.00'); + expect(results[sheets[1].id].B2.number).toBeCloseTo(7_140 / 15_900, 12); + for (const values of Object.values(results)) { + expect( + Object.entries(values).filter(([, value]) => value.error) + ).toEqual([]); + } + } finally { + engine.dispose(); + } + }); + + it('exports back to an .xlsx Excel reads with the same formulas', async () => { + const imported = await decodeXlsx(upgraded()); + + const exported = await encodeXlsx({ sheets: imported.sheets }); + + const external = new ExcelJS.Workbook(); + await external.xlsx.load(exported.bytes.slice().buffer); + const budget = external.getWorksheet('Budget')!; + expect(budget.getCell('B15').formula).toBe('SUM(B2:B13)'); + expect(budget.getCell('D2').formula).toBe('B2-C2'); + expect(budget.getCell('F1').isMerged).toBe(true); + expect(external.getWorksheet('Notes')!.getCell('B2').formula).toBe( + 'Budget!D15/Budget!B15' + ); + }); +}); diff --git a/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.test.ts b/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.test.ts index 9ee0512e56e..dc768e0cf0d 100644 --- a/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.test.ts +++ b/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.test.ts @@ -191,7 +191,7 @@ describe('workbook import and export actions', () => { it.each([ 'budget.xls', - 'budget.xlsm', + 'budget.xltm', 'budget.xlsx.exe', 'budget.csv', 'budget', @@ -222,6 +222,24 @@ describe('workbook import and export actions', () => { expect(actions.notice()).toContain('50 MB'); }); + it.each(['budget.xlsm', 'BUDGET.XLSM'])( + 'previews macro-enabled workbook %s like an .xlsx', + async (name) => { + const { actions } = setup(); + vi.mocked(importWorkbookFile).mockResolvedValue({ + ...imported, + warnings: ['VBA macros are not imported.'], + }); + await actions.importExcel({ ...file, name }); + expect(importWorkbookFile).toHaveBeenCalled(); + expect(actions.preview()?.name).toBe(name); + expect(actions.preview()?.data.warnings).toContain( + 'VBA macros are not imported.' + ); + expect(actions.notice()).toBe(''); + } + ); + it('replaces an old preview immediately and ignores stale decoder results from superseded requests', async () => { const { actions, store } = setup(); vi.mocked(importWorkbookFile).mockResolvedValueOnce(imported); diff --git a/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.ts b/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.ts index f8a5e44b5c3..1672e267f66 100644 --- a/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.ts +++ b/apps/web/src/features/block-spreadsheet/primitives/create-workbook-actions.ts @@ -78,9 +78,9 @@ export function createWorkbookActions(options: { } async function importExcel(file: File) { if (!options.store.canEdit()) return; - if (!/\.xlsx$/i.test(file.name)) { + if (!/\.xls[xm]$/i.test(file.name)) { setNotice( - 'Choose an .xlsx Excel workbook. Other file formats are not supported here.' + 'Choose an .xlsx or .xlsm Excel workbook. Other file formats are not supported here.' ); return; } diff --git a/apps/web/src/features/block-spreadsheet/queries/uploaded-workbook.test.ts b/apps/web/src/features/block-spreadsheet/queries/uploaded-workbook.test.ts index 555e9d8e9bd..f2b09ea3a60 100644 --- a/apps/web/src/features/block-spreadsheet/queries/uploaded-workbook.test.ts +++ b/apps/web/src/features/block-spreadsheet/queries/uploaded-workbook.test.ts @@ -32,6 +32,17 @@ describe('uploaded workbook retrieval', () => { expect(query().queryKey).not.toEqual(query('xlsx', 2).queryKey); expect(query().queryKey).not.toEqual(query('csv').queryKey); }); + it.each(['xlsm', 'XLSM'])( + 'decodes a macro-enabled %s upload with the Excel worker', + async (fileType) => { + const signal = new AbortController().signal; + vi.mocked(platformFetch).mockResolvedValueOnce( + new Response(new Uint8Array([4, 5])) + ); + await query(fileType).queryFn({ signal }); + expect(excel).toHaveBeenCalledWith(new Uint8Array([4, 5]), signal); + } + ); it('opens CSV text as safe typed cells, without invoking the Excel worker', async () => { vi.mocked(platformFetch).mockResolvedValueOnce( new Response('ID,Amount\r\n00123,12.5') diff --git a/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx b/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx index 42cdf952b13..ba01c686920 100644 --- a/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx +++ b/apps/web/src/features/block-spreadsheet/views/SpreadsheetEditor.tsx @@ -529,13 +529,18 @@ export function SpreadsheetEditor(props: { async function importFile(file: File | undefined) { if (!file || !editable()) return; - if (!/\.(csv|xlsx)$/i.test(file.name)) { + if (/\.xls$/i.test(file.name)) { + // Converting .xls to .xlsx needs the server; uploads are upgraded. actions.setNotice( - 'Choose a .csv or .xlsx file. Legacy .xls and macro-enabled workbooks are not supported.' + 'Legacy .xls workbooks are converted when uploaded. Upload the file, then choose Edit in Macro.' ); return; } - if (/\.xlsx$/i.test(file.name)) { + if (!/\.(csv|xlsx|xlsm)$/i.test(file.name)) { + actions.setNotice('Choose a .csv, .xlsx or .xlsm file.'); + return; + } + if (/\.xls[xm]$/i.test(file.name)) { await workbookActions.importExcel(file); return; } @@ -720,7 +725,7 @@ export function SpreadsheetEditor(props: { { diff --git a/apps/web/src/features/block-unknown/component/Block.tsx b/apps/web/src/features/block-unknown/component/Block.tsx index dc7355e4e43..897d74ae1fe 100644 --- a/apps/web/src/features/block-unknown/component/Block.tsx +++ b/apps/web/src/features/block-unknown/component/Block.tsx @@ -1,8 +1,10 @@ import { FileSidePanelSections, SidePanel } from '@components/app/side-panel'; +import { useSplitLayout } from '@components/app/split-layout/layout'; import { useBlockId } from '@core/block'; import { DocumentBlockContainer } from '@core/component/DocumentBlockContainer'; import { toast } from '@core/component/Toast/Toast'; import { useShareModal } from '@core/component/TopBar/shareModal'; +import { fileTypeToBlockName } from '@core/constant/allBlocks'; import { blockMetadataSignal } from '@core/signal/load'; import { useGetPermissions } from '@core/signal/permissions'; import { @@ -10,11 +12,26 @@ import { useBlockDocumentName, } from '@core/util/currentBlockDocumentName'; import { downloadFile } from '@filesystem/download'; +import { waitForDocumentContentReady } from '@queries/storage/document-location'; +import { fetchDocumentMetadata } from '@queries/storage/document-metadata'; import { createCallback } from '@solid-primitives/rootless'; -import { lazy, Show, Suspense } from 'solid-js'; +import { + createSignal, + lazy, + onCleanup, + onMount, + Show, + Suspense, +} from 'solid-js'; import { SpreadsheetSkeleton } from '../../block-spreadsheet/components/SpreadsheetSkeleton'; import { isUploadedWorkbook } from '../../block-spreadsheet/core/uploaded-workbook'; import { useSpreadsheetAccess } from '../../block-spreadsheet/primitives/use-spreadsheet-access'; +import { + legacyOfficeUpgradeTarget, + pollForLegacyUpgrade, + shouldAwaitLegacyUpgrade, + upgradedOfficeLabel, +} from '../legacy-office'; import { useGetFileBlob } from '../signal/blockData'; import { TopBar } from './TopBar'; import { UnknownContent } from './UnknownContent'; @@ -50,6 +67,7 @@ function BlockUnknownContent() { owner: blockMetadataSignal()?.owner, })); const getBlob = useGetFileBlob(); + const convertingTo = useLegacyOfficeUpgrade(blockId); const downloadDocument = createCallback(async () => { try { @@ -74,6 +92,7 @@ function BlockUnknownContent() { fallback={ void downloadDocument()} /> @@ -88,3 +107,56 @@ function BlockUnknownContent() { ); } + +/** + * Waits for a just-uploaded .doc/.ppt/.xls to be upgraded to OpenXML, then + * reopens it in the block for its new type. Returns the label of the format + * being converted to while waiting. + */ +function useLegacyOfficeUpgrade(documentId: string) { + const { replaceSplit } = useSplitLayout(); + const initial = blockMetadataSignal.get(); + const target = legacyOfficeUpgradeTarget(initial?.fileType); + const [waiting, setWaiting] = createSignal( + !!initial && !!target && shouldAwaitLegacyUpgrade(initial, Date.now()) + ); + + onMount(() => { + if (!target || !waiting()) return; + const controller = new AbortController(); + onCleanup(() => controller.abort()); + + void (async () => { + const upgraded = await pollForLegacyUpgrade({ + target, + fetchMetadata: () => fetchDocumentMetadata(documentId), + fileTypeOf: (metadata) => metadata.fileType, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + signal: controller.signal, + }); + if (controller.signal.aborted) return; + if (!upgraded) { + setWaiting(false); + return; + } + // Word content becomes readable once the DOCX pipeline processes it. + if (target === 'docx') { + await waitForDocumentContentReady({ + documentId, + timeoutMs: 60_000, + }).catch(() => undefined); + if (controller.signal.aborted) return; + } + const blockName = fileTypeToBlockName(upgraded.fileType); + if (blockName === 'unknown') { + // Excel workbooks stay in this block, which previews them. + blockMetadataSignal.set(upgraded); + setWaiting(false); + return; + } + replaceSplit({ content: { type: blockName, id: documentId } }); + })(); + }); + + return () => (waiting() && target ? upgradedOfficeLabel(target) : undefined); +} diff --git a/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx b/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx new file mode 100644 index 00000000000..25d6483c16c --- /dev/null +++ b/apps/web/src/features/block-unknown/component/UnknownContent.test.tsx @@ -0,0 +1,38 @@ +import { cleanup, render, screen } from '@solidjs/testing-library'; +import { afterEach, expect, it, vi } from 'vitest'; +import { UnknownContent } from './UnknownContent'; + +afterEach(cleanup); + +it('offers the file when there is no preview', () => { + render(() => ( + + )); + + expect(screen.getByText(/No preview available for/)).toBeTruthy(); + expect(screen.queryByTestId('legacy-office-converting')).toBeNull(); + expect(screen.getByText('Download')).toBeTruthy(); +}); + +it('says a legacy Office file is being converted and still offers the original', () => { + const onDownload = vi.fn(); + render(() => ( + + )); + + const status = screen.getByTestId('legacy-office-converting'); + expect(status.textContent).toContain('Board Update.ppt'); + expect(status.textContent).toContain('to a PowerPoint presentation'); + expect(screen.queryByText(/No preview available/)).toBeNull(); + screen.getByText('Download').click(); + expect(onDownload).toHaveBeenCalledOnce(); +}); diff --git a/apps/web/src/features/block-unknown/component/UnknownContent.tsx b/apps/web/src/features/block-unknown/component/UnknownContent.tsx index 98c9ee2c75e..64f44dfae2c 100644 --- a/apps/web/src/features/block-unknown/component/UnknownContent.tsx +++ b/apps/web/src/features/block-unknown/component/UnknownContent.tsx @@ -1,19 +1,41 @@ +import { LoadingSpinner } from '@core/component/LoadingSpinner'; import ShareFat from '@icon/share.svg'; import DownloadSimple from '@phosphor/download-simple.svg'; import { Button } from '@ui'; +import { Show } from 'solid-js'; export function UnknownContent(props: { fileName: string; + /** Set while a legacy Office file is being upgraded, e.g. "PowerPoint presentation". */ + convertingTo?: string; onShare: () => void; onDownload: () => void; }) { return (
-
- No preview available for{' '} - {props.fileName} -
+ + No preview available for{' '} + {props.fileName} +
+ } + > + {(convertingTo) => ( +
+ +
+ Converting {props.fileName}{' '} + to a {convertingTo()}… +
+
+ )} +