diff --git a/.changeset/funny-rocks-admire.md b/.changeset/funny-rocks-admire.md new file mode 100644 index 0000000000000..ec45e68f09e4b --- /dev/null +++ b/.changeset/funny-rocks-admire.md @@ -0,0 +1,5 @@ +--- +'@rocket.chat/meteor': patch +--- + +Fixes an issue where user data exports requested would remain stuck and never complete. diff --git a/apps/meteor/.mocharc.js b/apps/meteor/.mocharc.js index 6001bb77b7e58..f9f780024f964 100644 --- a/apps/meteor/.mocharc.js +++ b/apps/meteor/.mocharc.js @@ -13,6 +13,7 @@ module.exports = { 'lib/callbacks.spec.ts', 'server/lib/ldap/*.spec.ts', 'server/lib/ldap/**/*.spec.ts', + 'server/lib/dataExport/**/*.spec.ts', 'server/ufs/*.spec.ts', 'ee/server/lib/ldap/*.spec.ts', 'ee/tests/**/*.tests.ts', diff --git a/apps/meteor/server/lib/dataExport/exportRoomMessagesToFile.ts b/apps/meteor/server/lib/dataExport/exportRoomMessagesToFile.ts index 1faa7538d146b..5b46c07d2e179 100644 --- a/apps/meteor/server/lib/dataExport/exportRoomMessagesToFile.ts +++ b/apps/meteor/server/lib/dataExport/exportRoomMessagesToFile.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from 'fs/promises'; -import type { IMessage, IRoom, IUser, MessageAttachment, FileProp, RoomType } from '@rocket.chat/core-typings'; +import type { IMessage, IRoom, IUser, MessageAttachment, FileProp, RoomType, IExportOperation } from '@rocket.chat/core-typings'; import { Messages } from '@rocket.chat/models'; import { settings } from '../../../app/settings/server'; @@ -8,24 +8,16 @@ import { readSecondaryPreferred } from '../../database/readSecondaryPreferred'; import { joinPath } from '../fileUtils'; import { i18n } from '../i18n'; -const hideUserName = ( - username: string, - userData: Pick | undefined, - usersMap: { userNameTable: Record }, -) => { - if (!usersMap.userNameTable) { - usersMap.userNameTable = {}; - } - - if (!usersMap.userNameTable[username]) { +const hideUserName = (username: string, userData: Pick | undefined, usersMap: Record) => { + if (!usersMap[username]) { if (userData && username === userData.username) { - usersMap.userNameTable[username] = username; + usersMap[username] = username; } else { - usersMap.userNameTable[username] = `User_${Object.keys(usersMap.userNameTable).length + 1}`; + usersMap[username] = `User_${Object.keys(usersMap).length + 1}`; } } - return usersMap.userNameTable[username]; + return usersMap[username]; }; const getAttachmentData = (attachment: MessageAttachment, message: IMessage) => { @@ -66,7 +58,7 @@ export const getMessageData = ( msg: IMessage, hideUsers: boolean, userData: Pick | undefined, - usersMap: { userNameTable: Record }, + usersMap: IExportOperation['userNameTable'], ): MessageData => { const username = hideUsers ? hideUserName(msg.u.username || msg.u.name || '', userData, usersMap) : msg.u.username; @@ -199,7 +191,7 @@ export const exportRoomMessages = async ( limit: number, userData: any, filter: any = {}, - usersMap: any = {}, + usersMap: IExportOperation['userNameTable'] = {}, hideUsers = true, ) => { const readPreference = readSecondaryPreferred(); @@ -254,7 +246,7 @@ export const exportRoomMessagesToFile = async function ( )[], userData: IUser, messagesFilter = {}, - usersMap = {}, + usersMap: IExportOperation['userNameTable'] = {}, hideUsers = true, ) { await mkdir(exportPath, { recursive: true }); diff --git a/apps/meteor/server/lib/dataExport/processDataDownloads.spec.ts b/apps/meteor/server/lib/dataExport/processDataDownloads.spec.ts new file mode 100644 index 0000000000000..7f6f432b5cd25 --- /dev/null +++ b/apps/meteor/server/lib/dataExport/processDataDownloads.spec.ts @@ -0,0 +1,251 @@ +import fs from 'fs'; + +import type { IExportOperation } from '@rocket.chat/core-typings'; +import { expect } from 'chai'; +import proxyquire from 'proxyquire'; +import Sinon from 'sinon'; + +let exportOperation: IExportOperation | null = null; + +const modelsMock = { + ExportOperations: { + findLastOperationByUser: async (userId: string, fullExport = false) => { + if (exportOperation?.userId === userId && exportOperation?.fullExport === fullExport) { + return exportOperation; + } + }, + countAllPendingBeforeMyRequest: async (requestDay: Date) => { + if ( + exportOperation && + exportOperation.createdAt < requestDay && + exportOperation.status !== 'completed' && + exportOperation.status !== 'skipped' + ) { + return 1; + } + return 0; + }, + create: async (data: any) => { + exportOperation = { + userNameTable: null, // need to keep this null for testing purposes + ...data, + _id: 'exportOp1', + createdAt: new Date(), + }; + return exportOperation?._id as IExportOperation['_id']; + }, + updateOperation: async (data: IExportOperation) => { + if (exportOperation && exportOperation._id === data._id) { + exportOperation = { ...exportOperation, ...data }; + } + return { modifiedCount: 1 }; + }, + + findOnePending: async () => { + if (exportOperation && exportOperation.status !== 'completed' && exportOperation.status !== 'skipped') { + return exportOperation; + } + return null; + }, + }, + UserDataFiles: { + findOneById: async (fileId: string) => { + if (exportOperation?.fileId === fileId) { + return { + _id: fileId, + }; + } + }, + findLastFileByUser: async (userId: string) => { + if (exportOperation?.userId === userId && exportOperation.fileId) { + return { + _id: exportOperation.fileId, + }; + } + }, + }, + Avatars: { + findOneByName: async (_name: string) => { + return null; + }, + }, + Subscriptions: { + findByUserId: (_userId: string) => { + return [ + { + rid: 'general', + }, + ]; + }, + }, + Messages: { + findPaginated: (_query: object, _options: object) => { + return { + cursor: { + toArray: async () => [ + { + _id: 'msg1', + rid: 'general', + ts: new Date(), + msg: 'Hello World', + u: { _id: 'user1', username: 'userone' }, + }, + { + _id: 'msg2', + rid: 'general', + ts: new Date(), + msg: 'Second message', + u: { _id: 'user2', username: 'usertwo' }, + }, + ], + }, + totalCount: Promise.resolve(0), + }; + }, + }, +}; + +const { exportRoomMessagesToFile } = proxyquire.noCallThru().load('./exportRoomMessagesToFile.ts', { + '@rocket.chat/models': modelsMock, + '../../../app/settings/server': { + settings: { + get: (_key: string) => { + return undefined; + }, + }, + }, + '../i18n': { + i18n: { + t: (key: string) => key, + }, + }, +}); + +const { requestDataDownload } = proxyquire.noCallThru().load('../../methods/requestDataDownload.ts', { + '@rocket.chat/models': modelsMock, + '../../app/settings/server': { + settings: { + get: (_key: string) => { + return undefined; + }, + }, + }, + '../lib/dataExport': { + getPath: (fileId: string) => `/data-download/${fileId}`, + }, + 'meteor/meteor': { + Meteor: { + methods: Sinon.stub(), + }, + }, +}) as { + requestDataDownload: (args: { userData: { _id: string }; fullExport?: boolean }) => Promise<{ + requested: boolean; + exportOperation: IExportOperation; + url: string | null; + pendingOperationsBeforeMyRequest: number; + }>; +}; + +const { processDataDownloads } = proxyquire.noCallThru().load('./processDataDownloads.ts', { + '@rocket.chat/models': modelsMock, + '../../../app/file-upload/server': { + FileUpload: { + copy: async (fileId: string, _options: any) => { + return `copied-${fileId}`; + }, + }, + }, + '../../../app/settings/server': { + settings: { + get: (_key: string) => { + return undefined; + }, + }, + }, + '../../../app/utils/server/getURL': { + getURL: (path: string) => `https://example.com${path}`, + }, + '../i18n': { + i18n: { + t: (key: string) => key, + }, + }, + './copyFileUpload': { + copyFileUpload: (_attachmentData: { _id: string; name: string }, _assetsPath: string) => { + return Promise.resolve(); + }, + }, + './exportRoomMessagesToFile': { + exportRoomMessagesToFile, + }, + './getRoomData': { + getRoomData: Sinon.stub().resolves({ + roomId: 'GENERAL', + roomName: 'general', + type: 'c', + exportedCount: 0, + status: 'pending', + userId: 'user1', + targetFile: 'general.json', + }), + }, + './sendEmail': { + sendEmail: Sinon.stub().resolves(), + }, + './uploadZipFile': { + uploadZipFile: Sinon.stub().resolves({ _id: 'file1' }), + }, +}) as { + processDataDownloads: () => Promise; +}; + +const userData = { _id: 'user1', username: 'userone' }; + +describe('requestDataDownload', () => { + beforeEach(() => { + exportOperation = null; + }); + + it('should create a new export operation if none exists', async () => { + const result = await requestDataDownload({ userData, fullExport: false }); + + expect(result.requested).to.be.true; + expect(result.exportOperation).to.exist; + expect(result.exportOperation.userId).to.equal('user1'); + expect(result.exportOperation.fullExport).to.be.false; + expect(result.url).to.be.null; + expect(result.pendingOperationsBeforeMyRequest).to.equal(0); + expect(result.exportOperation.status).to.equal('pending'); + }); +}); + +describe('export user data', async () => { + beforeEach(() => { + exportOperation = null; + }); + it('should process data download for pending export operations', async () => { + await requestDataDownload({ userData, fullExport: true }); + + expect(exportOperation).to.not.be.null; + expect(exportOperation?.userId).to.equal('user1'); + expect(exportOperation?.fullExport).to.be.true; + expect(exportOperation?.status).to.equal('pending'); + + await processDataDownloads(); + + expect(exportOperation?.status).to.equal('completed'); + expect(exportOperation?.fileId).to.equal('file1'); + expect(exportOperation?.generatedUserFile).to.be.true; + expect(exportOperation?.roomList).to.have.lengthOf(1); + expect(exportOperation?.roomList?.[0].roomId).to.equal('GENERAL'); + expect(exportOperation?.roomList?.[0].exportedCount).to.equal(2); + expect(exportOperation?.exportPath).to.be.string; + + expect(fs.readFileSync(`${exportOperation?.exportPath}/${exportOperation?.roomList?.[0].targetFile}`, 'utf-8')).to.contain( + 'Hello World', + ); + expect(exportOperation?.generatedFile).to.be.string; + expect(fs.existsSync(exportOperation?.generatedFile as string)).to.be.true; + }); +}); diff --git a/apps/meteor/server/lib/dataExport/processDataDownloads.ts b/apps/meteor/server/lib/dataExport/processDataDownloads.ts index 1042e74b0422c..5a41c7d28713e 100644 --- a/apps/meteor/server/lib/dataExport/processDataDownloads.ts +++ b/apps/meteor/server/lib/dataExport/processDataDownloads.ts @@ -178,6 +178,9 @@ const continueExportOperation = async function (exportOperation: IExportOperatio // Run every room on every request, to avoid missing new messages on the rooms that finished first. if (exportOperation.status === 'exporting') { + if (!exportOperation.userNameTable) { + exportOperation.userNameTable = {}; + } const { fileList } = await exportRoomMessagesToFile( exportOperation.exportPath, exportOperation.assetsPath, diff --git a/apps/meteor/server/methods/requestDataDownload.ts b/apps/meteor/server/methods/requestDataDownload.ts index 6c489b3bf039d..6d49cb93dcb10 100644 --- a/apps/meteor/server/methods/requestDataDownload.ts +++ b/apps/meteor/server/methods/requestDataDownload.ts @@ -84,6 +84,7 @@ export const requestDataDownload = async ({ generatedFile: undefined, fullExport, userData: currentUserData, + userNameTable: {}, } as unknown as IExportOperation; // @todo yikes! const id = await ExportOperations.create(exportOperation);