diff --git a/app/importer-csv/server/adder.js b/app/importer-csv/server/adder.js deleted file mode 100644 index da1eaacdd0357..0000000000000 --- a/app/importer-csv/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { CsvImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { CsvImporterInfo } from '../lib/info'; - -Importers.add(new CsvImporterInfo(), CsvImporter); diff --git a/app/importer-csv/server/importer.js b/app/importer-csv/server/importer.js index 4e113e5c57264..66cc086c1c1ac 100644 --- a/app/importer-csv/server/importer.js +++ b/app/importer-csv/server/importer.js @@ -1,19 +1,11 @@ -import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; -import { Accounts } from 'meteor/accounts-base'; import { - RawImports, Base, ProgressStep, - Selection, - SelectionChannel, - SelectionUser, ImporterWebsocket, } from '../../importer/server'; -import { Users, Rooms } from '../../models'; -import { insertMessage } from '../../lib'; -import { t } from '../../utils'; +import { Users } from '../../models/server'; export class CsvImporter extends Base { constructor(info, importRecord) { @@ -24,20 +16,17 @@ export class CsvImporter extends Base { prepareUsingLocalFile(fullFilePath) { this.logger.debug('start preparing import operation'); - this.collection.remove({}); + this.converter.clearImportData(); const zip = new this.AdmZip(fullFilePath); const totalEntries = zip.getEntryCount(); ImporterWebsocket.progressUpdated({ rate: 0 }); - let tempChannels = []; - let tempUsers = []; - let hasDirectMessages = false; let count = 0; let oldRate = 0; - const increaseCount = () => { + const increaseProgressCount = () => { try { count++; const rate = Math.floor(count * 1000 / totalEntries) / 10; @@ -51,44 +40,93 @@ export class CsvImporter extends Base { }; let messagesCount = 0; + let usersCount = 0; + let channelsCount = 0; + const dmRooms = new Map(); + const roomIds = new Map(); + const usedUsernames = new Set(); + const availableUsernames = new Set(); + + const getRoomId = (roomName) => { + if (!roomIds.has(roomName)) { + roomIds.set(roomName, Random.id()); + } + + return roomIds.get(roomName); + }; + zip.forEach((entry) => { this.logger.debug(`Entry: ${ entry.entryName }`); // Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up if (entry.entryName.indexOf('__MACOSX') > -1) { this.logger.debug(`Ignoring the file: ${ entry.entryName }`); - return increaseCount(); + return increaseProgressCount(); } // Directories are ignored, since they are "virtual" in a zip file if (entry.isDirectory) { this.logger.debug(`Ignoring the directory entry: ${ entry.entryName }`); - return increaseCount(); + return increaseProgressCount(); } // Parse the channels if (entry.entryName.toLowerCase() === 'channels.csv') { super.updateProgress(ProgressStep.PREPARING_CHANNELS); const parsedChannels = this.csvParser(entry.getData().toString()); - tempChannels = parsedChannels.map((c) => ({ - id: Random.id(), - name: c[0].trim(), - creator: c[1].trim(), - isPrivate: c[2].trim().toLowerCase() === 'private', - members: c[3].trim().split(';').map((m) => m.trim()), - })); - return increaseCount(); + channelsCount = parsedChannels.length; + + for (const c of parsedChannels) { + const name = c[0].trim(); + const id = getRoomId(name); + const creator = c[1].trim(); + const isPrivate = c[2].trim().toLowerCase() === 'private'; + const members = c[3].trim().split(';').map((m) => m.trim()).filter((m) => m); + + this.converter.addChannel({ + importIds: [ + id, + ], + u: { + _id: creator, + }, + name, + users: members, + t: isPrivate ? 'p' : 'c', + }); + } + + super.updateRecord({ 'count.channels': channelsCount }); + return increaseProgressCount(); } // Parse the users if (entry.entryName.toLowerCase() === 'users.csv') { super.updateProgress(ProgressStep.PREPARING_USERS); const parsedUsers = this.csvParser(entry.getData().toString()); - tempUsers = parsedUsers.map((u) => ({ id: Random.id(), username: u[0].trim(), email: u[1].trim(), name: u[2].trim() })); - - super.updateRecord({ 'count.users': tempUsers.length }); + usersCount = parsedUsers.length; + + for (const u of parsedUsers) { + const username = u[0].trim(); + availableUsernames.add(username); + + const email = u[1].trim(); + const name = u[2].trim(); + + this.converter.addUser({ + importIds: [ + username, + ], + emails: [ + email, + ], + username, + name, + }); + } - return increaseCount(); + super.updateRecord({ 'count.users': parsedUsers.length }); + return increaseProgressCount(); } // Parse the messages @@ -106,14 +144,15 @@ export class CsvImporter extends Base { msgs = this.csvParser(entry.getData().toString()); } catch (e) { this.logger.warn(`The file ${ entry.entryName } contains invalid syntax`, e); - return increaseCount(); + return increaseProgressCount(); } let data; const msgGroupData = item[1].split('.')[0]; // messages + let isDirect = false; if (folderName.toLowerCase() === 'directmessages') { - hasDirectMessages = true; + isDirect = true; data = msgs.map((m) => ({ username: m[0], ts: m[2], text: m[3], otherUsername: m[1], isDirect: true })); } else { data = msgs.map((m) => ({ username: m[0], ts: m[1], text: m[2] })); @@ -124,370 +163,83 @@ export class CsvImporter extends Base { super.updateRecord({ messagesstatus: channelName }); - if (Base.getBSONSize(data) > Base.getMaxBSONSize()) { - Base.getBSONSafeArraysFromAnArray(data).forEach((splitMsg, i) => { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channelName }.${ i }`, messages: splitMsg, channel: folderName, i, msgGroupData }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: channelName, messages: data, channel: folderName, msgGroupData }); - } - - super.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); - return increaseCount(); - } - - increaseCount(); - }); - - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', users: tempUsers }); - super.addCountToTotal(messagesCount + tempUsers.length); - ImporterWebsocket.progressUpdated({ rate: 100 }); - - if (hasDirectMessages) { - tempChannels.push({ - id: '#directmessages#', - name: t('Direct_Messages'), - creator: 'rocket.cat', - isPrivate: false, - isDirect: true, - members: [], - }); - } - - // Insert the channels records. - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'channels', channels: tempChannels }); - super.updateRecord({ 'count.channels': tempChannels.length }); - super.addCountToTotal(tempChannels.length); - - // Ensure we have at least a single user, channel, or message - if (tempUsers.length === 0 && tempChannels.length === 0 && messagesCount === 0) { - this.logger.error('No users, channels, or messages found in the import file.'); - super.updateProgress(ProgressStep.ERROR); - return super.getProgress(); - } - - const selectionUsers = tempUsers.map((u) => new SelectionUser(u.id, u.username, u.email, false, false, true)); - const selectionChannels = tempChannels.map((c) => new SelectionChannel(c.id, c.name, false, true, c.isPrivate, undefined, c.isDirect)); - const selectionMessages = this.importRecord.count.messages; - - super.updateProgress(ProgressStep.USER_SELECTION); - return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); - } - - startImport(importSelection) { - this.users = RawImports.findOne({ import: this.importRecord._id, type: 'users' }); - this.channels = RawImports.findOne({ import: this.importRecord._id, type: 'channels' }); - this.reloadCount(); - - const rawCollection = this.collection.model.rawCollection(); - const distinct = Meteor.wrapAsync(rawCollection.distinct, rawCollection); - - super.startImport(importSelection); - const started = Date.now(); - - // Ensure we're only going to import the users that the user has selected - for (const user of importSelection.users) { - for (const u of this.users.users) { - if (u.id === user.user_id) { - u.do_import = user.do_import; - } - } - } - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - // Ensure we're only importing the channels the user has selected. - for (const channel of importSelection.channels) { - for (const c of this.channels.channels) { - if (c.id === channel.channel_id) { - c.do_import = channel.do_import; - } - } - } - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - const startedByUserId = Meteor.userId(); - Meteor.defer(() => { - super.updateProgress(ProgressStep.IMPORTING_USERS); - - try { - // Import the users - for (const u of this.users.users) { - if (!u.do_import) { - continue; - } - - Meteor.runAsUser(startedByUserId, () => { - let existantUser = Users.findOneByEmailAddress(u.email); - - // If we couldn't find one by their email address, try to find an existing user by their username - if (!existantUser) { - existantUser = Users.findOneByUsernameIgnoringCase(u.username); - } - - if (existantUser) { - // since we have an existing user, let's try a few things - u.rocketId = existantUser._id; - Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); - } else { - const userId = Accounts.createUser({ email: u.email, password: Date.now() + u.name + u.email.toUpperCase() }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', u.username, { joinDefaultChannelsSilenced: true }); - Users.setName(userId, u.name); - Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); - u.rocketId = userId; - }); - } - - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - // Import the channels - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - for (const c of this.channels.channels) { - if (!c.do_import) { - continue; - } - - if (c.isDirect) { - super.addCountCompleted(1); - continue; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantRoom = Rooms.findOneByName(c.name); - // If the room exists or the name of it is 'general', then we don't need to create it again - if (existantRoom || c.name.toUpperCase() === 'GENERAL') { - c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; - Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); - } else { - // Find the rocketchatId of the user who created this channel - let creatorId = startedByUserId; - for (const u of this.users.users) { - if (u.username === c.creator && u.do_import) { - creatorId = u.rocketId; - } - } - - // Create the channel - Meteor.runAsUser(creatorId, () => { - const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, c.members); - c.rocketId = roomInfo.rid; + if (isDirect) { + for (const msg of data) { + const sourceId = [msg.username, msg.otherUsername].sort().join('/'); + + if (!dmRooms.has(sourceId)) { + this.converter.addChannel({ + importIds: [ + sourceId, + ], + users: [msg.username, msg.otherUsername], + t: 'd', }); - Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); + dmRooms.set(sourceId, true); } - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - // If no channels file, collect channel map from DB for message-only import - if (this.channels.channels.length === 0) { - const channelNames = distinct('channel', { import: this.importRecord._id, type: 'messages', channel: { $ne: 'directMessages' } }); - for (const cname of channelNames) { - Meteor.runAsUser(startedByUserId, () => { - const existantRoom = Rooms.findOneByName(cname); - if (existantRoom || cname.toUpperCase() === 'GENERAL') { - this.channels.channels.push({ - id: cname.replace('.', '_'), - name: cname, - rocketId: cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id, - do_import: true, - }); - } - }); + const newMessage = { + rid: sourceId, + u: { + _id: msg.username, + }, + ts: new Date(parseInt(msg.ts)), + msg: msg.text, + }; + + usedUsernames.add(msg.username); + usedUsernames.add(msg.otherUsername); + this.converter.addMessage(newMessage); } - } - - // If no users file, collect user map from DB for message-only import - if (this.users.users.length === 0) { - const usernames = distinct('messages.username', { import: this.importRecord._id, type: 'messages' }); - for (const username of usernames) { - Meteor.runAsUser(startedByUserId, () => { - if (!this.getUserFromUsername(username)) { - const user = Users.findOneByUsernameIgnoringCase(username); - if (user) { - this.users.users.push({ - rocketId: user._id, - username: user.username, - }); - } - } - }); + } else { + const rid = getRoomId(folderName); + + for (const msg of data) { + const newMessage = { + rid, + u: { + _id: msg.username, + }, + ts: new Date(parseInt(msg.ts)), + msg: msg.text, + }; + + usedUsernames.add(msg.username); + this.converter.addMessage(newMessage); } } - // Import the Messages - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - - const messagePacks = this.collection.find({ import: this.importRecord._id, type: 'messages' }); - messagePacks.forEach((pack) => { - const ch = pack.channel; - const { msgGroupData } = pack; - - const csvChannel = this.getChannelFromName(ch); - if (!csvChannel || !csvChannel.do_import) { - return; - } - - if (csvChannel.isDirect) { - this._importDirectMessagesFile(msgGroupData, pack, startedByUserId); - return; - } - - if (ch.toLowerCase() === 'directmessages') { - return; - } - - const room = Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); - const timestamps = {}; - - Meteor.runAsUser(startedByUserId, () => { - super.updateRecord({ messagesstatus: `${ ch }/${ msgGroupData }.${ pack.messages.length }` }); - for (const msg of pack.messages) { - if (isNaN(new Date(parseInt(msg.ts)))) { - this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); - super.addCountCompleted(1); - continue; - } - - const creator = this.getUserFromUsername(msg.username); - if (creator) { - let suffix = ''; - if (timestamps[msg.ts] === undefined) { - timestamps[msg.ts] = 1; - } else { - suffix = `-${ timestamps[msg.ts] }`; - timestamps[msg.ts] += 1; - } - const msgObj = { - _id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`, - ts: new Date(parseInt(msg.ts)), - msg: msg.text, - rid: room._id, - u: { - _id: creator._id, - username: creator.username, - }, - }; - - insertMessage(creator, msgObj, room, true); - } - - super.addCountCompleted(1); - } - }); - }); - - super.updateProgress(ProgressStep.FINISHING); - super.updateProgress(ProgressStep.DONE); - } catch (e) { - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); + super.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); + return increaseProgressCount(); } - const timeTook = Date.now() - started; - this.logger.log(`CSV Import took ${ timeTook } milliseconds.`); + increaseProgressCount(); }); - return super.getProgress(); - } - - _importDirectMessagesFile(msgGroupData, msgs, startedByUserId) { - const dmUsers = {}; - - const findUser = (username) => { - if (!dmUsers[username]) { - const user = this.getUserFromUsername(username) || Users.findOneByUsername(username, { fields: { username: 1 } }); - dmUsers[username] = user; - } - - return dmUsers[username]; - }; - - Meteor.runAsUser(startedByUserId, () => { - const timestamps = {}; - let room; - let rid; - super.updateRecord({ messagesstatus: `${ t('Direct_Messagest') }/${ msgGroupData }.${ msgs.messages.length }` }); - for (const msg of msgs.messages) { - if (isNaN(new Date(parseInt(msg.ts)))) { - this.logger.warn(`Timestamp on a message in ${ t('Direct_Messagest') }/${ msgGroupData } is invalid`); - super.addCountCompleted(1); - continue; - } - - const creator = findUser(msg.username); - const targetUser = findUser(msg.otherUsername); - - if (creator && targetUser) { - if (!rid) { - const roomInfo = Meteor.runAsUser(creator._id, () => Meteor.call('createDirectMessage', targetUser.username)); - rid = roomInfo.rid; - room = Rooms.findOneById(rid, { fields: { usernames: 1, t: 1, name: 1 } }); - } - - if (!room) { - this.logger.warn(`DM room not found for users ${ msg.username } and ${ msg.otherUsername }`); - super.addCountCompleted(1); - continue; - } - - let suffix = ''; - if (timestamps[msg.ts] === undefined) { - timestamps[msg.ts] = 1; - } else { - suffix = `-${ timestamps[msg.ts] }`; - timestamps[msg.ts] += 1; - } - - const msgObj = { - _id: `csv-${ rid }-${ msg.ts }${ suffix }`, - ts: new Date(parseInt(msg.ts)), - msg: msg.text, - rid: room._id, - u: { - _id: creator._id, - username: creator.username, - }, - }; - - insertMessage(creator, msgObj, room, true); - } - - super.addCountCompleted(1); + // Check if any of the message usernames was not in the imported list of users + for (const username of usedUsernames) { + if (availableUsernames.has(username)) { + continue; } - }); - } - - getChannelFromName(channelName) { - if (channelName.toLowerCase() === 'directmessages') { - return this.getDirectMessagesChannel(); - } - for (const ch of this.channels.channels) { - if (ch.name === channelName) { - return ch; + // Check if an user with that username already exists + const user = Users.findOneByUsername(username); + if (user && !user.importIds?.includes(username)) { + // Add the username to the local user's importIds so it can be found by the import process + // This way we can support importing new messages for existing users + Users.addImportIds(user._id, username); } } - } - getDirectMessagesChannel() { - for (const ch of this.channels.channels) { - if (ch.is_direct || ch.isDirect) { - return ch; - } - } - } + super.addCountToTotal(messagesCount + usersCount + channelsCount); + ImporterWebsocket.progressUpdated({ rate: 100 }); - getUserFromUsername(username) { - for (const u of this.users.users) { - if (u.username === username) { - return Users.findOneById(u.rocketId, { fields: { username: 1 } }); - } + // Ensure we have at least a single user, channel, or message + if (usersCount === 0 && channelsCount === 0 && messagesCount === 0) { + this.logger.error('No users, channels, or messages found in the import file.'); + super.updateProgress(ProgressStep.ERROR); + return super.getProgress(); } } } diff --git a/app/importer-csv/server/index.js b/app/importer-csv/server/index.js index 44a1b3bab84c5..da1eaacdd0357 100644 --- a/app/importer-csv/server/index.js +++ b/app/importer-csv/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { CsvImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { CsvImporterInfo } from '../lib/info'; + +Importers.add(new CsvImporterInfo(), CsvImporter); diff --git a/app/importer-hipchat-enterprise/server/adder.js b/app/importer-hipchat-enterprise/server/adder.js deleted file mode 100644 index 11f9a8e7b4b65..0000000000000 --- a/app/importer-hipchat-enterprise/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { HipChatEnterpriseImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { HipChatEnterpriseImporterInfo } from '../lib/info'; - -Importers.add(new HipChatEnterpriseImporterInfo(), HipChatEnterpriseImporter); diff --git a/app/importer-hipchat-enterprise/server/importer.js b/app/importer-hipchat-enterprise/server/importer.js index 6203150012709..3f59b669e6323 100644 --- a/app/importer-hipchat-enterprise/server/importer.js +++ b/app/importer-hipchat-enterprise/server/importer.js @@ -2,21 +2,13 @@ import { Readable } from 'stream'; import path from 'path'; import fs from 'fs'; -import limax from 'limax'; import { Meteor } from 'meteor/meteor'; -import { Accounts } from 'meteor/accounts-base'; -import { Random } from 'meteor/random'; import TurndownService from 'turndown'; import { Base, ProgressStep, - Selection, - SelectionChannel, - SelectionUser, } from '../../importer/server'; -import { Messages, Users, Subscriptions, Rooms, Imports } from '../../models'; -import { insertMessage } from '../../lib'; const turndownService = new TurndownService({ strongDelimiter: '*', @@ -43,8 +35,6 @@ export class HipChatEnterpriseImporter extends Base { this.tarStream = require('tar-stream'); this.extract = this.tarStream.extract(); this.path = path; - - this.emailList = []; } parseData(data) { @@ -58,274 +48,173 @@ export class HipChatEnterpriseImporter extends Base { } } - async storeTempUsers(tempUsers) { - await this.collection.model.rawCollection().update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - }, { - $set: { - import: this.importRecord._id, - importer: this.name, - type: 'users', - }, - $push: { - users: { $each: tempUsers }, - }, - }, { - upsert: true, - }); - - this.usersCount += tempUsers.length; - } - async prepareUsersFile(file) { super.updateProgress(ProgressStep.PREPARING_USERS); - let tempUsers = []; let count = 0; for (const u of file) { - const userData = { - id: u.User.id, - email: u.User.email, - name: u.User.name, + const newUser = { + emails: [], + importIds: [ + String(u.User.id), + ], username: u.User.mention_name, - avatar: u.User.avatar && u.User.avatar.replace(/\n/g, ''), - timezone: u.User.timezone, - isDeleted: u.User.is_deleted, + name: u.User.name, + avatarUrl: u.User.avatar && `data:image/png;base64,${ u.User.avatar.replace(/\n/g, '') }`, + bio: u.User.title || undefined, + deleted: u.User.is_deleted, + type: 'user', }; count++; if (u.User.email) { - if (this.emailList.indexOf(u.User.email) >= 0) { - userData.is_email_taken = true; - } else { - this.emailList.push(u.User.email); - } - } - - tempUsers.push(userData); - if (tempUsers.length >= 100) { - await this.storeTempUsers(tempUsers); // eslint-disable-line no-await-in-loop - tempUsers = []; + newUser.emails.push(u.User.email); } - } - if (tempUsers.length > 0) { - this.storeTempUsers(tempUsers); + this.converter.addUser(newUser); } super.updateRecord({ 'count.users': count }); super.addCountToTotal(count); } - async storeTempRooms(tempRooms) { - await this.collection.model.rawCollection().update({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }, { - $set: { - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }, - $push: { - channels: { $each: tempRooms }, - }, - }, { - upsert: true, - }); - - this.channelsCount += tempRooms.length; - } - async prepareRoomsFile(file) { super.updateProgress(ProgressStep.PREPARING_CHANNELS); - let tempRooms = []; let count = 0; for (const r of file) { - tempRooms.push({ - id: r.Room.id, - creator: r.Room.owner, - created: new Date(r.Room.created), + this.converter.addChannel({ + u: { + _id: r.Room.owner, + }, + importIds: [ + String(r.Room.id), + ], name: r.Room.name, - isPrivate: r.Room.privacy === 'private', - isArchived: r.Room.is_archived, + users: r.Room.members, + t: r.Room.privacy === 'private' ? 'p' : 'c', topic: r.Room.topic, - members: r.Room.members, + ts: new Date(r.Room.created), + archived: r.Room.is_archived, }); - count++; - - if (tempRooms.length >= 100) { - await this.storeTempRooms(tempRooms); // eslint-disable-line no-await-in-loop - tempRooms = []; - } - } - if (tempRooms.length > 0) { - await this.storeTempRooms(tempRooms); + count++; } super.updateRecord({ 'count.channels': count }); super.addCountToTotal(count); } - async storeTempMessages(tempMessages, roomIdentifier, index, subIndex, hipchatRoomId) { - this.logger.debug('dumping messages to database'); - const name = subIndex ? `${ roomIdentifier }/${ index }/${ subIndex }` : `${ roomIdentifier }/${ index }`; - - await this.collection.model.rawCollection().insert({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - name, - messages: tempMessages, - roomIdentifier, - hipchatRoomId, - }); - - this.messagesCount += tempMessages.length; - } - - async storeUserTempMessages(tempMessages, roomIdentifier, index) { - this.logger.debug(`dumping ${ tempMessages.length } messages from room ${ roomIdentifier } to database`); - await this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'user-messages', - name: `${ roomIdentifier }/${ index }`, - messages: tempMessages, - roomIdentifier, - }); - - this.messagesCount += tempMessages.length; - } - - async prepareUserMessagesFile(file, roomIdentifier, index) { - await this.loadExistingMessagesIfNecessary(); - let msgs = []; + async prepareUserMessagesFile(file) { this.logger.debug(`preparing room with ${ file.length } messages `); - for (const m of file) { - if (m.PrivateUserMessage) { - // If the message id is already on the list, skip it - if (this.preparedMessages[m.PrivateUserMessage.id] !== undefined) { - continue; - } - this.preparedMessages[m.PrivateUserMessage.id] = true; + let count = 0; + const dmRooms = []; - const newId = `hipchatenterprise-private-${ m.PrivateUserMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - const skipAttachment = skipMessage && (m.PrivateUserMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true); + for (const m of file) { + if (!m.PrivateUserMessage) { + continue; + } - if (!skipMessage || !skipAttachment) { - msgs.push({ - type: 'user', - id: newId, - senderId: m.PrivateUserMessage.sender.id, - receiverId: m.PrivateUserMessage.receiver.id, - text: m.PrivateUserMessage.message.indexOf('/me ') === -1 ? m.PrivateUserMessage.message : `${ m.PrivateUserMessage.message.replace(/\/me /, '_') }_`, + // If the message id is already on the list, skip it + if (this.preparedMessages[m.PrivateUserMessage.id] !== undefined) { + continue; + } + this.preparedMessages[m.PrivateUserMessage.id] = true; + + const senderId = String(m.PrivateUserMessage.sender.id); + const receiverId = String(m.PrivateUserMessage.receiver.id); + const users = [senderId, receiverId].sort(); + + if (!dmRooms[receiverId]) { + dmRooms[receiverId] = this.converter.findDMForImportedUsers(senderId, receiverId); + + if (!dmRooms[receiverId]) { + const room = { + importIds: [ + users.join(''), + ], + users, + t: 'd', ts: new Date(m.PrivateUserMessage.timestamp.split(' ')[0]), - attachment: m.PrivateUserMessage.attachment, - attachment_path: m.PrivateUserMessage.attachment_path, - skip: skipMessage, - skipAttachment, - }); + }; + this.converter.addChannel(room); + dmRooms[receiverId] = room; } } - if (msgs.length >= 500) { - await this.storeUserTempMessages(msgs, roomIdentifier, index); // eslint-disable-line no-await-in-loop - msgs = []; - } - } - - if (msgs.length > 0) { - await this.storeUserTempMessages(msgs, roomIdentifier, index); + const rid = dmRooms[receiverId].importIds[0]; + const newMessage = this.convertImportedMessage(m.PrivateUserMessage, rid, 'private'); + count++; + this.converter.addMessage(newMessage); } - return msgs.length; + return count; } - _checkIfMessageExists(messageId) { - if (this._hasAnyImportedMessage === false) { - return false; - } + convertImportedMessage(importedMessage, rid, type) { + const idType = type === 'private' ? type : `${ rid }-${ type }`; + const newId = `hipchatenterprise-${ idType }-${ importedMessage.id }`; - return this._previewsMessagesIds.has(messageId); - } + const newMessage = { + _id: newId, + rid, + ts: new Date(importedMessage.timestamp.split(' ')[0]), + u: { + _id: String(importedMessage.sender.id), + }, + }; - async loadExistingMessagesIfNecessary() { - if (this._hasAnyImportedMessage === false) { - return false; + const text = importedMessage.message; + + if (importedMessage.message_format === 'html') { + newMessage.msg = turndownService.turndown(text); + } else if (text.startsWith('/me ')) { + newMessage.msg = `${ text.replace(/\/me /, '_') }_`; + } else { + newMessage.msg = text; } - if (!this._previewsMessagesIds) { - this._previewsMessagesIds = new Set(); - await Messages.model.rawCollection().find({}, { fields: { _id: 1 } }).forEach((i) => this._previewsMessagesIds.add(i._id)); + if (importedMessage.attachment?.url) { + const fileId = `${ importedMessage.id }-${ importedMessage.attachment.name || 'attachment' }`; + + newMessage._importFile = { + downloadUrl: importedMessage.attachment.url, + id: `${ fileId }`, + size: importedMessage.attachment.size || 0, + name: importedMessage.attachment.name, + external: false, + source: 'hipchat-enterprise', + original: { + ...importedMessage.attachment, + }, + }; } + + return newMessage; } - async prepareRoomMessagesFile(file, roomIdentifier, id, index) { - let roomMsgs = []; + async prepareRoomMessagesFile(file, rid) { this.logger.debug(`preparing room with ${ file.length } messages `); - let subIndex = 0; - - await this.loadExistingMessagesIfNecessary(); + let count = 0; for (const m of file) { if (m.UserMessage) { - const newId = `hipchatenterprise-${ id }-user-${ m.UserMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - const skipAttachment = skipMessage && (m.UserMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true); - - if (!skipMessage || !skipAttachment) { - roomMsgs.push({ - type: 'user', - id: newId, - userId: m.UserMessage.sender.id, - text: m.UserMessage.message.indexOf('/me ') === -1 ? m.UserMessage.message : `${ m.UserMessage.message.replace(/\/me /, '_') }_`, - ts: new Date(m.UserMessage.timestamp.split(' ')[0]), - attachment: m.UserMessage.attachment, - attachment_path: m.UserMessage.attachment_path, - skip: skipMessage, - skipAttachment, - }); - } + const newMessage = this.convertImportedMessage(m.UserMessage, rid, 'user'); + this.converter.addMessage(newMessage); + count++; } else if (m.NotificationMessage) { - const text = m.NotificationMessage.message.indexOf('/me ') === -1 ? m.NotificationMessage.message : `${ m.NotificationMessage.message.replace(/\/me /, '_') }_`; - const newId = `hipchatenterprise-${ id }-notif-${ m.NotificationMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - const skipAttachment = skipMessage && (m.NotificationMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true); + const newMessage = this.convertImportedMessage(m.NotificationMessage, rid, 'notif'); + newMessage.u._id = 'rocket.cat'; + newMessage.alias = m.NotificationMessage.sender; - if (!skipMessage || !skipAttachment) { - roomMsgs.push({ - type: 'user', - id: newId, - userId: 'rocket.cat', - alias: m.NotificationMessage.sender, - text: m.NotificationMessage.message_format === 'html' ? turndownService.turndown(text) : text, - ts: new Date(m.NotificationMessage.timestamp.split(' ')[0]), - attachment: m.NotificationMessage.attachment, - attachment_path: m.NotificationMessage.attachment_path, - skip: skipMessage, - skipAttachment, - }); - } + this.converter.addMessage(newMessage); + count++; } else if (m.TopicRoomMessage) { - const newId = `hipchatenterprise-${ id }-topic-${ m.TopicRoomMessage.id }`; - const skipMessage = this._checkIfMessageExists(newId); - if (!skipMessage) { - roomMsgs.push({ - type: 'topic', - id: newId, - userId: m.TopicRoomMessage.sender.id, - ts: new Date(m.TopicRoomMessage.timestamp.split(' ')[0]), - text: m.TopicRoomMessage.message, - skip: skipMessage, - }); - } + const newMessage = this.convertImportedMessage(m.TopicRoomMessage, rid, 'topic'); + newMessage.t = 'room_changed_topic'; + + this.converter.addMessage(newMessage); + count++; } else if (m.ArchiveRoomMessage) { this.logger.warn('Archived Room Notification was ignored.'); } else if (m.GuestAccessMessage) { @@ -333,38 +222,24 @@ export class HipChatEnterpriseImporter extends Base { } else { this.logger.error('HipChat Enterprise importer isn\'t configured to handle this message:', m); } - - if (roomMsgs.length >= 500) { - subIndex++; - await this.storeTempMessages(roomMsgs, roomIdentifier, index, subIndex, id); // eslint-disable-line no-await-in-loop - roomMsgs = []; - } - } - - if (roomMsgs.length > 0) { - await this.storeTempMessages(roomMsgs, roomIdentifier, index, subIndex > 0 ? subIndex + 1 : undefined, id); } - return roomMsgs.length; + return count; } async prepareMessagesFile(file, info) { super.updateProgress(ProgressStep.PREPARING_MESSAGES); - let messageGroupIndex = 0; - let userMessageGroupIndex = 0; - const [type, id] = info.dir.split('/'); // ['users', '1'] + const [type, id] = info.dir.split('/'); const roomIdentifier = `${ type }/${ id }`; super.updateRecord({ messagesstatus: roomIdentifier }); switch (type) { case 'users': - userMessageGroupIndex++; - return this.prepareUserMessagesFile(file, roomIdentifier, userMessageGroupIndex); + return this.prepareUserMessagesFile(file); case 'rooms': - messageGroupIndex++; - return this.prepareRoomMessagesFile(file, roomIdentifier, id, messageGroupIndex); + return this.prepareRoomMessagesFile(file, id); default: this.logger.error(`HipChat Enterprise importer isn't configured to handle "${ type }" files (${ info.dir }).`); return 0; @@ -388,215 +263,24 @@ export class HipChatEnterpriseImporter extends Base { case 'history.json': return this.prepareMessagesFile(file, info); case 'emoticons.json': - this.logger.error('HipChat Enterprise importer doesn\'t import emoticons.', info); + case 'metadata.json': break; default: - this.logger.error(`HipChat Enterprise importer doesn't know what to do with the file "${ fileName }" :o`, info); + this.logger.error(`HipChat Enterprise importer doesn't know what to do with the file "${ fileName }"`); break; } return 0; } - async _prepareFolderEntry(fullEntryPath, relativeEntryPath) { - const files = fs.readdirSync(fullEntryPath); - for (const fileName of files) { - try { - const fullFilePath = path.join(fullEntryPath, fileName); - const fullRelativePath = path.join(relativeEntryPath, fileName); - - this.logger.info(`new entry from import folder: ${ fileName }`); - - if (fs.statSync(fullFilePath).isDirectory()) { - await this._prepareFolderEntry(fullFilePath, fullRelativePath); // eslint-disable-line no-await-in-loop - continue; - } - - if (!fileName.endsWith('.json')) { - continue; - } - - let fileData; - - const promise = new Promise((resolve, reject) => { - fs.readFile(fullFilePath, (error, data) => { - if (error) { - this.logger.error(error); - return reject(error); - } - - fileData = data; - return resolve(); - }); - }); - - await promise.catch((error) => { // eslint-disable-line no-await-in-loop - this.logger.error(error); - fileData = null; - }); - - if (!fileData) { - this.logger.info(`Skipping the file: ${ fileName }`); - continue; - } - - this.logger.info(`Processing the file: ${ fileName }`); - const info = this.path.parse(fullRelativePath); - await this.prepareFile(info, fileData, fileName); // eslint-disable-line no-await-in-loop - - this.logger.debug('moving to next import folder entry'); - } catch (e) { - this.logger.debug('failed to prepare file'); - this.logger.error(e); - } - } - } - - prepareUsingLocalFolder(fullFolderPath) { - this.logger.debug('start preparing import operation using local folder'); - this.collection.remove({}); - this.emailList = []; - - this._hasAnyImportedMessage = Boolean(Messages.findOne({ _id: /hipchatenterprise\-.*/ })); - - this.usersCount = 0; - this.channelsCount = 0; - this.messagesCount = 0; - - // HipChat duplicates direct messages (one for each user) - // This object will keep track of messages that have already been prepared so it doesn't try to do it twice - this.preparedMessages = {}; - - const promise = new Promise(async (resolve, reject) => { - try { - await this._prepareFolderEntry(fullFolderPath, '.'); - this._finishPreparationProcess(resolve, reject); - } catch (e) { - this.logger.error(e); - reject(e); - } - }); - - return promise; - } - - async _finishPreparationProcess(resolve, reject) { - await this.fixPublicChannelMembers(); - - this.logger.info('finished parsing files, checking for errors now'); - this._previewsMessagesIds = undefined; - this.emailList = []; - this.preparedMessages = {}; - - - super.updateRecord({ 'count.messages': this.messagesCount, messagesstatus: null }); - super.addCountToTotal(this.messagesCount); - - // Check if any of the emails used are already taken - if (this.emailList.length > 0) { - const conflictingUsers = Users.find({ 'emails.address': { $in: this.emailList } }); - const conflictingUserEmails = []; - - conflictingUsers.forEach((conflictingUser) => { - if (conflictingUser.emails && conflictingUser.emails.length) { - conflictingUser.emails.forEach((email) => { - conflictingUserEmails.push(email.address); - }); - } - }); - - if (conflictingUserEmails.length > 0) { - this.flagConflictingEmails(conflictingUserEmails); - } - } - - // Ensure we have some users, channels, and messages - if (!this.usersCount && !this.channelsCount && !this.messagesCount) { - this.logger.info(`users: ${ this.usersCount }, channels: ${ this.channelsCount }, messages = ${ this.messagesCount }`); - super.updateProgress(ProgressStep.ERROR); - reject(new Meteor.Error('error-import-file-is-empty')); - return; - } - - const tempUsers = this.collection.findOne({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - }); - - const tempChannels = this.collection.findOne({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }); - - const selectionUsers = tempUsers.users.map((u) => new SelectionUser(u.id, u.username, u.email, u.isDeleted, false, u.do_import !== false, u.is_email_taken === true)); - const selectionChannels = tempChannels.channels.map((r) => new SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate, r.creator)); - const selectionMessages = this.messagesCount; - - super.updateProgress(ProgressStep.USER_SELECTION); - - resolve(new Selection(this.name, selectionUsers, selectionChannels, selectionMessages)); - } - - async fixPublicChannelMembers() { - await this.collection.model.rawCollection().aggregate([{ - $match: { - import: this.importRecord._id, - type: 'channels', - }, - }, { - $unwind: '$channels', - }, { - $match: { - 'channels.members.0': { $exists: false }, - }, - }, { - $group: { _id: '$channels.id' }, - }]).forEach(async (channel) => { - const userIds = (await this.collection.model.rawCollection().aggregate([{ - $match: { - $or: [ - { roomIdentifier: `rooms/${ channel._id }` }, - { roomIdentifier: `users/${ channel._id }` }, - ], - }, - }, { - $unwind: '$messages', - }, { - $match: { 'messages.userId': { $ne: 'rocket.cat' } }, - }, { - $group: { _id: '$messages.userId' }, - }]).toArray()).map((i) => i._id); - - await this.collection.model.rawCollection().update({ - 'channels.id': channel._id, - }, { - $set: { - 'channels.$.members': userIds, - }, - }); - }); - } - prepareUsingLocalFile(fullFilePath) { - if (fs.statSync(fullFilePath).isDirectory()) { - return this.prepareUsingLocalFolder(fullFilePath); - } - this.logger.debug('start preparing import operation'); - this.collection.remove({}); - this.emailList = []; - - this._hasAnyImportedMessage = Boolean(Messages.findOne({ _id: /hipchatenterprise\-.*/ })); - - this.usersCount = 0; - this.channelsCount = 0; - this.messagesCount = 0; + this.converter.clearImportData(); // HipChat duplicates direct messages (one for each user) // This object will keep track of messages that have already been prepared so it doesn't try to do it twice this.preparedMessages = {}; + let messageCount = 0; const promise = new Promise((resolve, reject) => { this.extract.on('entry', Meteor.bindEnvironment((header, stream, next) => { @@ -617,7 +301,12 @@ export class HipChatEnterpriseImporter extends Base { stream.on('end', Meteor.bindEnvironment(async () => { this.logger.info(`Processing the file: ${ header.name }`); - await this.prepareFile(info, data, header.name); + const newMessageCount = await this.prepareFile(info, data, header.name); + + messageCount += newMessageCount; + super.updateRecord({ 'count.messages': messageCount }); + super.addCountToTotal(newMessageCount); + data = undefined; this.logger.debug('next import entry'); @@ -634,7 +323,7 @@ export class HipChatEnterpriseImporter extends Base { }); this.extract.on('finish', Meteor.bindEnvironment(() => { - this._finishPreparationProcess(resolve, reject); + resolve(); })); const rs = fs.createReadStream(fullFilePath); @@ -650,664 +339,4 @@ export class HipChatEnterpriseImporter extends Base { return promise; } - - _saveUserIdReference(hipchatId, rocketId) { - this._userIdReference[hipchatId] = rocketId; - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - 'users.id': hipchatId, - }, { - $set: { - 'users.$.rocketId': rocketId, - }, - }); - } - - _getUserRocketId(hipchatId) { - if (!this._userIdReference) { - return; - } - - return this._userIdReference[hipchatId]; - } - - _saveRoomIdReference(hipchatId, rocketId) { - this._roomIdReference[hipchatId] = rocketId; - } - - _getRoomRocketId(hipchatId) { - if (!this._roomIdReference) { - return; - } - - return this._roomIdReference[hipchatId]; - } - - _updateImportedUser(userToImport, existingUserId) { - userToImport.rocketId = existingUserId; - this._saveUserIdReference(userToImport.id, existingUserId); - - Meteor.runAsUser(existingUserId, () => { - Users.update({ _id: existingUserId }, { - $push: { - importIds: userToImport.id, - }, - $set: { - active: userToImport.isDeleted !== true, - name: userToImport.name, - username: userToImport.username, - }, - }); - - // TODO: Think about using a custom field for the users "title" field - if (userToImport.avatar) { - Meteor.call('setAvatarFromService', `data:image/png;base64,${ userToImport.avatar }`); - } - }); - } - - _importUser(userToImport, startedByUserId) { - Meteor.runAsUser(startedByUserId, () => { - let existingUser = Users.findOneByUsernameIgnoringCase(userToImport.username); - if (!existingUser) { - // If there's no user with that username, but there's an imported user with the same original ID and no username, use that - existingUser = Users.findOne({ - importIds: userToImport.id, - username: { $exists: false }, - }); - } - - if (existingUser) { - // since we have an existing user, let's try a few things - this._saveUserIdReference(userToImport.id, existingUser._id); - userToImport.rocketId = existingUser._id; - - try { - this._updateImportedUser(userToImport, existingUser._id); - } catch (e) { - this.logger.error(e); - this.addUserError(userToImport.id, e); - } - } else { - const user = { - email: userToImport.email, - password: Random.id(), - username: userToImport.username, - name: userToImport.name, - active: userToImport.isDeleted !== true, - }; - if (!user.email) { - delete user.email; - } - if (!user.username) { - delete user.username; - } - if (!user.name) { - delete user.name; - } - - try { - const userId = Accounts.createUser(user); - - userToImport.rocketId = userId; - this._saveUserIdReference(userToImport.id, userId); - - this._updateImportedUser(userToImport, userId); - } catch (e) { - this.logger.error(e); - this.addUserError(userToImport.id, e); - } - } - - super.addCountCompleted(1); - }); - } - - _applyUserSelections(importSelection) { - // Ensure we're only going to import the users that the user has selected - const usersToImport = importSelection.users.filter((user) => user.do_import !== false).map((user) => user.user_id); - const usersNotToImport = importSelection.users.filter((user) => user.do_import === false).map((user) => user.user_id); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - 'users.id': { - $in: usersToImport, - }, - }, { - $set: { - 'users.$.do_import': true, - }, - }); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - 'users.id': { - $in: usersNotToImport, - }, - }, { - $set: { - 'users.$.do_import': false, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.users.user_id': { - $in: usersToImport, - }, - }, { - $set: { - 'fileData.users.$.do_import': true, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.users.user_id': { - $in: usersNotToImport, - }, - }, { - $set: { - 'fileData.users.$.do_import': false, - }, - }); - - // Ensure we're only importing the channels the user has selected. - const channelsToImport = importSelection.channels.filter((channel) => channel.do_import !== false).map((channel) => channel.channel_id); - const channelsNotToImport = importSelection.channels.filter((channel) => channel.do_import === false).map((channel) => channel.channel_id); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - 'channels.id': { - $in: channelsToImport, - }, - }, { - $set: { - 'channels.$.do_import': true, - }, - }); - - this.collection.update({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - 'channels.id': { - $in: channelsNotToImport, - }, - }, { - $set: { - 'channels.$.do_import': false, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.channels.channel_id': { - $in: channelsToImport, - }, - }, { - $set: { - 'fileData.channels.$.do_import': true, - }, - }); - - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.channels.channel_id': { - $in: channelsNotToImport, - }, - }, { - $set: { - 'fileData.channels.$.do_import': false, - }, - }); - } - - startImport(importSelection) { - this.reloadCount(); - super.startImport(importSelection); - this._userDataCache = {}; - const started = Date.now(); - - this._applyUserSelections(importSelection); - - const startedByUserId = Meteor.userId(); - Meteor.defer(async () => { - try { - await super.updateProgress(ProgressStep.IMPORTING_USERS); - await this._importUsers(startedByUserId); - - await super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - await this._importChannels(startedByUserId); - - await super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - await this._importMessages(startedByUserId); - await this._importDirectMessages(); - - // super.updateProgress(ProgressStep.FINISHING); - await super.updateProgress(ProgressStep.DONE); - } catch (e) { - super.updateRecord({ 'error-record': JSON.stringify(e, Object.getOwnPropertyNames(e)) }); - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); - } - - const timeTook = Date.now() - started; - this.logger.log(`HipChat Enterprise Import took ${ timeTook } milliseconds.`); - this._userDataCache = {}; - this._userIdReference = {}; - this._roomIdReference = {}; - }); - - return super.getProgress(); - } - - _importUsers(startedByUserId) { - this._userIdReference = {}; - - const userLists = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - }); - - userLists.forEach((list) => { - if (!list.users) { - return; - } - - list.users.forEach((u) => { - this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`); - if (u.do_import === false) { - return; - } - - this._importUser(u, startedByUserId); - }); - }); - } - - _createSubscriptions(channelToImport, roomOrRoomId) { - if (!channelToImport || !channelToImport.members) { - return; - } - - let room; - if (roomOrRoomId && typeof roomOrRoomId === 'string') { - room = Rooms.findOneByIdOrName(roomOrRoomId); - } else { - room = roomOrRoomId; - } - - const extra = { open: true }; - channelToImport.members.forEach((hipchatUserId) => { - if (hipchatUserId === channelToImport.creator) { - // Creators are subscribed automatically - return; - } - - const user = this.getRocketUserFromUserId(hipchatUserId); - if (!user) { - this.logger.error(`User ${ hipchatUserId } not found on Rocket.Chat database.`); - return; - } - - if (Subscriptions.find({ rid: room._id, 'u._id': user._id }, { limit: 1 }).count() === 0) { - this.logger.info(`Creating user's subscription to room ${ room._id }, rocket.chat user is ${ user._id }, hipchat user is ${ hipchatUserId }`); - Subscriptions.createWithRoomAndUser(room, user, extra); - } - }); - } - - _importChannel(channelToImport, startedByUserId) { - Meteor.runAsUser(startedByUserId, () => { - const existingRoom = Rooms.findOneByName(limax(channelToImport.name)); - // If the room exists or the name of it is 'general', then we don't need to create it again - if (existingRoom || channelToImport.name.toUpperCase() === 'GENERAL') { - channelToImport.rocketId = channelToImport.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existingRoom._id; - this._saveRoomIdReference(channelToImport.id, channelToImport.rocketId); - Rooms.update({ _id: channelToImport.rocketId }, { $push: { importIds: channelToImport.id } }); - - this._createSubscriptions(channelToImport, existingRoom || 'general'); - } else { - // Find the rocketchatId of the user who created this channel - const creatorId = this._getUserRocketId(channelToImport.creator) || startedByUserId; - - // Create the channel - Meteor.runAsUser(creatorId, () => { - try { - const roomInfo = Meteor.call(channelToImport.isPrivate ? 'createPrivateGroup' : 'createChannel', channelToImport.name, []); - this._saveRoomIdReference(channelToImport.id, roomInfo.rid); - channelToImport.rocketId = roomInfo.rid; - } catch (e) { - this.logger.error(`Failed to create channel, using userId: ${ creatorId };`, e); - } - }); - - if (channelToImport.rocketId) { - Rooms.update({ _id: channelToImport.rocketId }, { $set: { ts: channelToImport.created, topic: channelToImport.topic }, $push: { importIds: channelToImport.id } }); - this._createSubscriptions(channelToImport, channelToImport.rocketId); - } - } - - super.addCountCompleted(1); - }); - } - - _importChannels(startedByUserId) { - this._roomIdReference = {}; - const channelLists = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - }); - - channelLists.forEach((list) => { - if (!list.channels) { - return; - } - list.channels.forEach((c) => { - this.logger.debug(`Starting the channel import: ${ c.name } and are we importing them? ${ c.do_import }`); - if (c.do_import === false) { - return; - } - - this._importChannel(c, startedByUserId); - }); - }); - } - - _importAttachment(msg, room, sender) { - if (msg.attachment_path && !msg.skipAttachment) { - const details = { - message_id: `${ msg.id }-attachment`, - name: msg.attachment.name, - size: msg.attachment.size, - userId: sender._id, - rid: room._id, - }; - - this.uploadFile(details, msg.attachment.url, sender, room, msg.ts); - } - } - - _importSingleMessage(msg, roomIdentifier, room) { - if (isNaN(msg.ts)) { - this.logger.error(`Timestamp on a message in ${ roomIdentifier } is invalid`); - return; - } - - try { - const creator = this.getRocketUserFromUserId(msg.userId); - if (creator) { - Meteor.runAsUser(creator._id, () => { - this._importAttachment(msg, room, creator); - - switch (msg.type) { - case 'user': - if (!msg.skip) { - insertMessage(creator, { - _id: msg.id, - ts: msg.ts, - msg: msg.text, - rid: room._id, - alias: msg.alias, - u: { - _id: creator._id, - username: creator.username, - }, - }, room, false); - } - break; - case 'topic': - Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, msg.text, creator, { _id: msg.id, ts: msg.ts }); - break; - } - }); - } else { - this.logger.error(`Hipchat user not found: ${ msg.userId }`); - this.addMessageError(new Meteor.Error('error-message-sender-is-invalid'), `Hipchat user not found: ${ msg.userId }`); - } - } catch (e) { - this.logger.error(e); - this.addMessageError(e, msg); - } - } - - async _importMessageList(startedByUserId, messageListId) { - const list = this.collection.findOneById(messageListId); - if (!list) { - return; - } - - if (!list.messages) { - return; - } - - const { roomIdentifier, hipchatRoomId, name } = list; - const rid = await this._getRoomRocketId(hipchatRoomId); - - // If there's no rocketId for the channel, then it wasn't imported - if (!rid) { - this.logger.debug(`Ignoring room ${ roomIdentifier } ( ${ name } ), as there's no rid to use.`); - return; - } - - const room = await Rooms.findOneById(rid, { fields: { usernames: 1, t: 1, name: 1 } }); - await super.updateRecord({ - messagesstatus: `${ roomIdentifier }.${ list.messages.length }`, - 'count.completed': this.progress.count.completed, - }); - - await Meteor.runAsUser(startedByUserId, async () => { - let msgCount = 0; - try { - for (const msg of list.messages) { - await this._importSingleMessage(msg, roomIdentifier, room); // eslint-disable-line no-await-in-loop - msgCount++; - if (msgCount >= 50) { - super.addCountCompleted(msgCount); - msgCount = 0; - } - } - } catch (e) { - this.logger.error(e); - } - - if (msgCount > 0) { - super.addCountCompleted(msgCount); - } - }); - } - - async _importMessages(startedByUserId) { - const messageListIds = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - }, { fields: { _id: true } }).fetch(); - - for (const item of messageListIds) { - await this._importMessageList(startedByUserId, item._id); // eslint-disable-line no-await-in-loop - } - } - - _importDirectMessages() { - const messageListIds = this.collection.find({ - import: this.importRecord._id, - importer: this.name, - type: 'user-messages', - }, { fields: { _id: true } }).fetch(); - - this.logger.info(`${ messageListIds.length } lists of messages to import.`); - - // HipChat duplicates direct messages (one for each user) - // This object will keep track of messages that have already been imported so it doesn't try to insert them twice - const importedMessages = {}; - - messageListIds.forEach((item) => { - this.logger.debug(`New list of user messages: ${ item._id }`); - const list = this.collection.findOneById(item._id); - if (!list) { - this.logger.error('Record of user-messages list not found'); - return; - } - - if (!list.messages) { - this.logger.error('No message list found on record.'); - return; - } - - const { roomIdentifier } = list; - if (!this.getRocketUserFromRoomIdentifier(roomIdentifier)) { - this.logger.error(`Skipping ${ list.messages.length } messages due to missing room ( ${ roomIdentifier } ).`); - return; - } - - this.logger.debug(`${ list.messages.length } messages on this list`); - super.updateRecord({ - messagesstatus: `${ list.name }.${ list.messages.length }`, - 'count.completed': this.progress.count.completed, - }); - - let msgCount = 0; - const roomUsers = {}; - const roomObjects = {}; - - list.messages.forEach((msg) => { - msgCount++; - if (isNaN(msg.ts)) { - this.logger.error(`Timestamp on a message in ${ list.name } is invalid`); - return; - } - - // make sure the message sender is a valid user inside rocket.chat - if (!(msg.senderId in roomUsers)) { - roomUsers[msg.senderId] = this.getRocketUserFromUserId(msg.senderId); - } - - if (!roomUsers[msg.senderId]) { - this.logger.error(`Skipping message due to missing sender ( ${ msg.senderId } ).`); - return; - } - - // make sure the receiver of the message is a valid rocket.chat user - if (!(msg.receiverId in roomUsers)) { - roomUsers[msg.receiverId] = this.getRocketUserFromUserId(msg.receiverId); - } - - if (!roomUsers[msg.receiverId]) { - this.logger.error(`Skipping message due to missing receiver ( ${ msg.receiverId } ).`); - return; - } - - const sender = roomUsers[msg.senderId]; - const receiver = roomUsers[msg.receiverId]; - - const roomId = [receiver._id, sender._id].sort().join(''); - if (!(roomId in roomObjects)) { - roomObjects[roomId] = Rooms.findOneById(roomId); - } - - let room = roomObjects[roomId]; - if (!room) { - this.logger.debug('DM room not found, creating it.'); - Meteor.runAsUser(sender._id, () => { - const roomInfo = Meteor.call('createDirectMessage', receiver.username); - - room = Rooms.findOneById(roomInfo.rid); - roomObjects[roomId] = room; - }); - } - - try { - Meteor.runAsUser(sender._id, () => { - if (importedMessages[msg.id] !== undefined) { - return; - } - importedMessages[msg.id] = true; - - if (msg.attachment_path) { - if (!msg.skipAttachment) { - this.logger.debug('Uploading DM file'); - const details = { - message_id: `${ msg.id }-attachment`, - name: msg.attachment.name, - size: msg.attachment.size, - userId: sender._id, - rid: room._id, - }; - this.uploadFile(details, msg.attachment.url, sender, room, msg.ts); - } - } - - if (!msg.skip) { - this.logger.debug('Inserting DM message'); - insertMessage(sender, { - _id: msg.id, - ts: msg.ts, - msg: msg.text, - rid: room._id, - u: { - _id: sender._id, - username: sender.username, - }, - }, room, false); - } - }); - } catch (e) { - console.error(e); - this.addMessageError(e, msg); - } - - if (msgCount >= 50) { - super.addCountCompleted(msgCount); - msgCount = 0; - } - }); - - if (msgCount > 0) { - super.addCountCompleted(msgCount); - } - }); - } - - _getBasicUserData(userId) { - if (this._userDataCache[userId]) { - return this._userDataCache[userId]; - } - - this._userDataCache[userId] = Users.findOneById(userId, { fields: { username: 1 } }); - return this._userDataCache[userId]; - } - - getRocketUserFromUserId(userId) { - if (userId === 'rocket.cat') { - return this._getBasicUserData('rocket.cat'); - } - - const rocketId = this._getUserRocketId(userId); - if (rocketId) { - return this._getBasicUserData(rocketId); - } - } - - getRocketUserFromRoomIdentifier(roomIdentifier) { - const userParts = roomIdentifier.split('/'); - if (!userParts || !userParts.length) { - return; - } - - const userId = userParts[userParts.length - 1]; - return this.getRocketUserFromUserId(userId); - } } diff --git a/app/importer-hipchat-enterprise/server/index.js b/app/importer-hipchat-enterprise/server/index.js index 44a1b3bab84c5..11f9a8e7b4b65 100644 --- a/app/importer-hipchat-enterprise/server/index.js +++ b/app/importer-hipchat-enterprise/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { HipChatEnterpriseImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { HipChatEnterpriseImporterInfo } from '../lib/info'; + +Importers.add(new HipChatEnterpriseImporterInfo(), HipChatEnterpriseImporter); diff --git a/app/importer-hipchat/client/adder.js b/app/importer-hipchat/client/adder.js deleted file mode 100644 index 813169cc932e8..0000000000000 --- a/app/importer-hipchat/client/adder.js +++ /dev/null @@ -1,4 +0,0 @@ -import { Importers } from '../../importer/client'; -import { HipChatImporterInfo } from '../lib/info'; - -Importers.add(new HipChatImporterInfo()); diff --git a/app/importer-hipchat/client/index.js b/app/importer-hipchat/client/index.js deleted file mode 100644 index 44a1b3bab84c5..0000000000000 --- a/app/importer-hipchat/client/index.js +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/app/importer-hipchat/lib/info.js b/app/importer-hipchat/lib/info.js deleted file mode 100644 index befd126868dc2..0000000000000 --- a/app/importer-hipchat/lib/info.js +++ /dev/null @@ -1,7 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class HipChatImporterInfo extends ImporterInfo { - constructor() { - super('hipchat', 'HipChat (zip)', 'application/zip'); - } -} diff --git a/app/importer-hipchat/server/adder.js b/app/importer-hipchat/server/adder.js deleted file mode 100644 index 7379752371af3..0000000000000 --- a/app/importer-hipchat/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { HipChatImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { HipChatImporterInfo } from '../lib/info'; - -Importers.add(new HipChatImporterInfo(), HipChatImporter); diff --git a/app/importer-hipchat/server/importer.js b/app/importer-hipchat/server/importer.js deleted file mode 100644 index acf521f2305d4..0000000000000 --- a/app/importer-hipchat/server/importer.js +++ /dev/null @@ -1,375 +0,0 @@ -import limax from 'limax'; -import { Meteor } from 'meteor/meteor'; -import { Accounts } from 'meteor/accounts-base'; -import _ from 'underscore'; -import moment from 'moment'; - -import { - RawImports, - Base, - ProgressStep, - Selection, - SelectionChannel, - SelectionUser, -} from '../../importer/server'; -import { RocketChatFile } from '../../file'; -import { Users, Rooms } from '../../models'; -import { sendMessage } from '../../lib'; - -import 'moment-timezone'; - -export class HipChatImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - - this.userTags = []; - this.roomPrefix = 'hipchat_export/rooms/'; - this.usersPrefix = 'hipchat_export/users/'; - } - - prepare(dataURI, sentContentType, fileName, skipTypeCheck) { - super.prepare(dataURI, sentContentType, fileName, skipTypeCheck); - const { image } = RocketChatFile.dataURIParse(dataURI); - // const contentType = ref.contentType; - const zip = new this.AdmZip(Buffer.from(image, 'base64')); - const zipEntries = zip.getEntries(); - let tempRooms = []; - let tempUsers = []; - const tempMessages = {}; - - zipEntries.forEach((entry) => { - if (entry.entryName.indexOf('__MACOSX') > -1) { - this.logger.debug(`Ignoring the file: ${ entry.entryName }`); - } - if (entry.isDirectory) { - return; - } - if (entry.entryName.indexOf(this.roomPrefix) > -1) { - let roomName = entry.entryName.split(this.roomPrefix)[1]; - if (roomName === 'list.json') { - super.updateProgress(ProgressStep.PREPARING_CHANNELS); - tempRooms = JSON.parse(entry.getData().toString()).rooms; - tempRooms.forEach((room) => { - room.name = limax(room.name); - }); - } else if (roomName.indexOf('/') > -1) { - const item = roomName.split('/'); - roomName = limax(item[0]); - const msgGroupData = item[1].split('.')[0]; - if (!tempMessages[roomName]) { - tempMessages[roomName] = {}; - } - try { - tempMessages[roomName][msgGroupData] = JSON.parse(entry.getData().toString()); - return tempMessages[roomName][msgGroupData]; - } catch (error) { - return this.logger.warn(`${ entry.entryName } is not a valid JSON file! Unable to import it.`); - } - } - } else if (entry.entryName.indexOf(this.usersPrefix) > -1) { - const usersName = entry.entryName.split(this.usersPrefix)[1]; - if (usersName === 'list.json') { - super.updateProgress(ProgressStep.PREPARING_USERS); - tempUsers = JSON.parse(entry.getData().toString()).users; - return tempUsers; - } - return this.logger.warn(`Unexpected file in the ${ this.name } import: ${ entry.entryName }`); - } - }); - const usersId = this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'users', - users: tempUsers, - }); - this.users = this.collection.findOne(usersId); - this.updateRecord({ - 'count.users': tempUsers.length, - }); - this.addCountToTotal(tempUsers.length); - const channelsId = this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'channels', - channels: tempRooms, - }); - this.channels = this.collection.findOne(channelsId); - this.updateRecord({ - 'count.channels': tempRooms.length, - }); - this.addCountToTotal(tempRooms.length); - super.updateProgress(ProgressStep.PREPARING_MESSAGES); - let messagesCount = 0; - - Object.keys(tempMessages).forEach((channel) => { - const messagesObj = tempMessages[channel]; - - Object.keys(messagesObj).forEach((date) => { - const msgs = messagesObj[date]; - messagesCount += msgs.length; - this.updateRecord({ - messagesstatus: `${ channel }/${ date }`, - }); - - if (Base.getBSONSize(msgs) > Base.getMaxBSONSize()) { - Base.getBSONSafeArraysFromAnArray(msgs).forEach((splitMsg, i) => { - this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - name: `${ channel }/${ date }.${ i }`, - messages: splitMsg, - channel, - date, - i, - }); - }); - } else { - this.collection.insert({ - import: this.importRecord._id, - importer: this.name, - type: 'messages', - name: `${ channel }/${ date }`, - messages: msgs, - channel, - date, - }); - } - }); - }); - this.updateRecord({ - 'count.messages': messagesCount, - messagesstatus: null, - }); - this.addCountToTotal(messagesCount); - if (tempUsers.length === 0 || tempRooms.length === 0 || messagesCount === 0) { - this.logger.warn(`The loaded users count ${ tempUsers.length }, the loaded channels ${ tempRooms.length }, and the loaded messages ${ messagesCount }`); - super.updateProgress(ProgressStep.ERROR); - return this.getProgress(); - } - const selectionUsers = tempUsers.map(function(user) { - return new SelectionUser(user.user_id, user.name, user.email, user.is_deleted, false, !user.is_bot); - }); - const selectionChannels = tempRooms.map(function(room) { - return new SelectionChannel(room.room_id, room.name, room.is_archived, true, false); - }); - const selectionMessages = this.importRecord.count.messages; - super.updateProgress(ProgressStep.USER_SELECTION); - return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); - } - - startImport(importSelection) { - this.users = RawImports.findOne({ import: this.importRecord._id, type: 'users' }); - this.channels = RawImports.findOne({ import: this.importRecord._id, type: 'channels' }); - this.reloadCount(); - - super.startImport(importSelection); - const start = Date.now(); - - importSelection.users.forEach((user) => { - this.users.users.forEach((u) => { - if (u.user_id === user.user_id) { - u.do_import = user.do_import; - } - }); - }); - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - importSelection.channels.forEach((channel) => - this.channels.channels.forEach((c) => { - if (c.room_id === channel.channel_id) { - c.do_import = channel.do_import; - } - }), - ); - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - const startedByUserId = Meteor.userId(); - Meteor.defer(() => { - super.updateProgress(ProgressStep.IMPORTING_USERS); - - try { - this.users.users.forEach((user) => { - if (!user.do_import) { - return; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantUser = Users.findOneByEmailAddress(user.email); - if (existantUser) { - user.rocketId = existantUser._id; - this.userTags.push({ - hipchat: `@${ user.mention_name }`, - rocket: `@${ existantUser.username }`, - }); - } else { - const userId = Accounts.createUser({ - email: user.email, - password: Date.now() + user.name + user.email.toUpperCase(), - }); - user.rocketId = userId; - this.userTags.push({ - hipchat: `@${ user.mention_name }`, - rocket: `@${ user.mention_name }`, - }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', user.mention_name, { - joinDefaultChannelsSilenced: true, - }); - Meteor.call('setAvatarFromService', user.photo_url, undefined, 'url'); - return Meteor.call('userSetUtcOffset', parseInt(moment().tz(user.timezone).format('Z').toString().split(':')[0])); - }); - if (user.name != null) { - Users.setName(userId, user.name); - } - if (user.is_deleted) { - Meteor.call('setUserActiveStatus', userId, false); - } - } - return this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); - - const channelNames = []; - - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - this.channels.channels.forEach((channel) => { - if (!channel.do_import) { - return; - } - - channelNames.push(channel.name); - Meteor.runAsUser(startedByUserId, () => { - channel.name = channel.name.replace(/ /g, ''); - const existantRoom = Rooms.findOneByName(channel.name); - if (existantRoom) { - channel.rocketId = existantRoom._id; - } else { - let userId = ''; - this.users.users.forEach((user) => { - if (user.user_id === channel.owner_user_id) { - userId = user.rocketId; - } - }); - if (userId === '') { - this.logger.warn(`Failed to find the channel creator for ${ channel.name }, setting it to the current running user.`); - userId = startedByUserId; - } - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createChannel', channel.name, []); - channel.rocketId = returned.rid; - }); - Rooms.update({ - _id: channel.rocketId, - }, { - $set: { - ts: new Date(channel.created * 1000), - }, - }); - } - return this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); - - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - const nousers = {}; - - for (const channel of channelNames) { - const hipchatChannel = this.getHipChatChannelFromName(channel); - - if (!hipchatChannel || !hipchatChannel.do_import) { - continue; - } - - const room = Rooms.findOneById(hipchatChannel.rocketId, { - fields: { - usernames: 1, - t: 1, - name: 1, - }, - }); - - const messagePacks = this.collection.find({ import: this.importRecord._id, type: 'messages', channel }); - - Meteor.runAsUser(startedByUserId, () => { - messagePacks.forEach((pack) => { - const packId = pack.i ? `${ pack.date }.${ pack.i }` : pack.date; - - this.updateRecord({ messagesstatus: `${ channel }/${ packId } (${ pack.messages.length })` }); - pack.messages.forEach((message) => { - if (message.from != null) { - const user = this.getRocketUser(message.from.user_id); - if (user != null) { - const msgObj = { - msg: this.convertHipChatMessageToRocketChat(message.message), - ts: new Date(message.date), - u: { - _id: user._id, - username: user.username, - }, - }; - sendMessage(user, msgObj, room, true); - } else if (!nousers[message.from.user_id]) { - nousers[message.from.user_id] = message.from; - } - } else if (!_.isArray(message)) { - console.warn('Please report the following:', message); - } - - this.addCountCompleted(1); - }); - }); - }); - } - - this.logger.warn('The following did not have users:', nousers); - super.updateProgress(ProgressStep.FINISHING); - - this.channels.channels.forEach((channel) => { - if (channel.do_import && channel.is_archived) { - Meteor.runAsUser(startedByUserId, () => Meteor.call('archiveRoom', channel.rocketId)); - } - }); - - super.updateProgress(ProgressStep.DONE); - } catch (e) { - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); - } - - const timeTook = Date.now() - start; - return this.logger.log(`Import took ${ timeTook } milliseconds.`); - }); - - return this.getProgress(); - } - - getHipChatChannelFromName(channelName) { - return this.channels.channels.find((channel) => channel.name === channelName); - } - - getRocketUser(hipchatId) { - const user = this.users.users.find((user) => user.user_id === hipchatId); - return user ? Users.findOneById(user.rocketId, { - fields: { - username: 1, - name: 1, - }, - }) : undefined; - } - - convertHipChatMessageToRocketChat(message) { - if (message != null) { - this.userTags.forEach((userReplace) => { - message = message.replace(userReplace.hipchat, userReplace.rocket); - }); - } else { - message = ''; - } - return message; - } -} diff --git a/app/importer-hipchat/server/index.js b/app/importer-hipchat/server/index.js deleted file mode 100644 index 44a1b3bab84c5..0000000000000 --- a/app/importer-hipchat/server/index.js +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/app/importer-pending-avatars/server/importer.js b/app/importer-pending-avatars/server/importer.js index 5d034a8c7d371..7a8767ba582af 100644 --- a/app/importer-pending-avatars/server/importer.js +++ b/app/importer-pending-avatars/server/importer.js @@ -8,12 +8,6 @@ import { import { Users } from '../../models'; export class PendingAvatarImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - this.userTags = []; - this.bots = {}; - } - prepareFileCount() { this.logger.debug('start preparing import operation'); super.updateProgress(ProgressStep.PREPARING_STARTED); diff --git a/app/importer-slack-users/server/adder.js b/app/importer-slack-users/server/adder.js deleted file mode 100644 index 1651465e5d4e3..0000000000000 --- a/app/importer-slack-users/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { SlackUsersImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { SlackUsersImporterInfo } from '../lib/info'; - -Importers.add(new SlackUsersImporterInfo(), SlackUsersImporter); diff --git a/app/importer-slack-users/server/index.js b/app/importer-slack-users/server/index.js index 44a1b3bab84c5..1651465e5d4e3 100644 --- a/app/importer-slack-users/server/index.js +++ b/app/importer-slack-users/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { SlackUsersImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { SlackUsersImporterInfo } from '../lib/info'; + +Importers.add(new SlackUsersImporterInfo(), SlackUsersImporter); diff --git a/app/importer-slack/server/adder.js b/app/importer-slack/server/adder.js deleted file mode 100644 index d8499b517f006..0000000000000 --- a/app/importer-slack/server/adder.js +++ /dev/null @@ -1,5 +0,0 @@ -import { SlackImporter } from './importer'; -import { Importers } from '../../importer/server'; -import { SlackImporterInfo } from '../lib/info'; - -Importers.add(new SlackImporterInfo(), SlackImporter); diff --git a/app/importer-slack/server/importer.js b/app/importer-slack/server/importer.js index a996985ff06ff..6aeb3fd1343ea 100644 --- a/app/importer-slack/server/importer.js +++ b/app/importer-slack/server/importer.js @@ -1,30 +1,17 @@ -import { Meteor } from 'meteor/meteor'; -import { Accounts } from 'meteor/accounts-base'; import _ from 'underscore'; import { - RawImports, Base, ProgressStep, - Selection, - SelectionChannel, - SelectionUser, + ImportData, ImporterWebsocket, } from '../../importer/server'; -import { getUserAvatarURL } from '../../utils/lib/getUserAvatarURL'; -import { Users, Rooms, Messages } from '../../models'; -import { insertMessage, createDirectRoom } from '../../lib'; -import { getValidRoomName } from '../../utils'; +import { Messages } from '../../models'; import { settings } from '../../settings/server'; import { MentionsParser } from '../../mentions/lib/MentionsParser'; +import { getUserAvatarURL } from '../../utils/lib/getUserAvatarURL'; export class SlackImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - this.userTags = []; - this.bots = {}; - } - parseData(data) { const dataString = data.toString(); try { @@ -36,107 +23,238 @@ export class SlackImporter extends Base { } } + prepareChannelsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + + this.logger.debug(`loaded ${ data.length } channels.`); + + this.addCountToTotal(data.length); + + for (const channel of data) { + this.converter.addChannel({ + _id: channel.is_general ? 'general' : undefined, + u: { + _id: this._replaceSlackUserId(channel.creator), + }, + importIds: [ + channel.id, + ], + name: channel.name, + users: this._replaceSlackUserIds(channel.members), + t: 'c', + topic: channel.topic?.value || undefined, + description: channel.purpose?.value || undefined, + ts: channel.created ? new Date(channel.created * 1000) : undefined, + archived: channel.is_archived, + }); + } + + return data.length; + } + + prepareGroupsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + + this.logger.debug(`loaded ${ data.length } groups.`); + + this.addCountToTotal(data.length); + + for (const channel of data) { + this.converter.addChannel({ + u: { + _id: this._replaceSlackUserId(channel.creator), + }, + importIds: [ + channel.id, + ], + name: channel.name, + users: this._replaceSlackUserIds(channel.members), + t: 'p', + topic: channel.topic?.value || undefined, + description: channel.purpose?.value || undefined, + ts: channel.created ? new Date(channel.created * 1000) : undefined, + archived: channel.is_archived, + }); + } + + return data.length; + } + + prepareMpimpsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + + this.logger.debug(`loaded ${ data.length } mpims.`); + + this.addCountToTotal(data.length); + + const maxUsers = settings.get('DirectMesssage_maxUsers') || 1; + + for (const channel of data) { + this.converter.addChannel({ + u: { + _id: this._replaceSlackUserId(channel.creator), + }, + importIds: [ + channel.id, + ], + name: channel.name, + users: this._replaceSlackUserIds(channel.members), + t: channel.members.length > maxUsers ? 'p' : 'd', + topic: channel.topic?.value || undefined, + description: channel.purpose?.value || undefined, + ts: channel.created ? new Date(channel.created * 1000) : undefined, + archived: channel.is_archived, + }); + } + + return data.length; + } + + prepareDMsFile(entry) { + super.updateProgress(ProgressStep.PREPARING_CHANNELS); + const data = JSON.parse(entry.getData().toString()); + + this.logger.debug(`loaded ${ data.length } dms.`); + + this.addCountToTotal(data.length); + for (const channel of data) { + this.converter.addChannel({ + importIds: [ + channel.id, + ], + users: this._replaceSlackUserIds(channel.members), + t: 'd', + ts: channel.created ? new Date(channel.created * 1000) : undefined, + }); + } + + return data.length; + } + + prepareUsersFile(entry) { + super.updateProgress(ProgressStep.PREPARING_USERS); + const data = JSON.parse(entry.getData().toString()); + + this.logger.debug(`loaded ${ data.length } users.`); + + // Insert the users record + this.updateRecord({ 'count.users': data.length }); + this.addCountToTotal(data.length); + + + for (const user of data) { + const newUser = { + emails: [], + importIds: [ + user.id, + ], + username: user.name, + name: user.profile.real_name, + utcOffset: user.tz_offset && (user.tz_offset / 3600), + avatarUrl: user.profile.image_original || user.profile.image_512, + deleted: user.deleted, + statusText: user.profile.status_text || undefined, + bio: user.profile.title || undefined, + type: 'user', + }; + + if (user.profile.email) { + newUser.emails.push(user.profile.email); + } + + if (user.is_bot) { + newUser.roles = ['bot']; + newUser.type = 'bot'; + } + + this.converter.addUser(newUser); + } + } + prepareUsingLocalFile(fullFilePath) { this.logger.debug('start preparing import operation'); - this.collection.remove({}); + this.converter.clearImportData(); const zip = new this.AdmZip(fullFilePath); const totalEntries = zip.getEntryCount(); - let tempChannels = []; - let tempGroups = []; - let tempMpims = []; - let tempDMs = []; - let tempUsers = []; let messagesCount = 0; + let channelCount = 0; let count = 0; ImporterWebsocket.progressUpdated({ rate: 0 }); let oldRate = 0; - const prepareChannelsFile = (entry, typeName, filterInvalidCreators = true) => { - super.updateProgress(ProgressStep.PREPARING_CHANNELS); - let data = JSON.parse(entry.getData().toString()); - - if (filterInvalidCreators) { - data = data.filter((channel) => channel.creator != null); - } - - this.logger.debug(`loaded ${ data.length } ${ typeName }.`); - - // Insert the channels records. - if (Base.getBSONSize(data) > Base.getMaxBSONSize()) { - const tmp = Base.getBSONSafeArraysFromAnArray(data); - Object.keys(tmp).forEach((i) => { - const splitChannels = tmp[i]; - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: typeName, name: `${ typeName }/${ i }`, channels: splitChannels, i }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: typeName, channels: data }); + const increaseProgress = () => { + try { + count++; + const rate = Math.floor(count * 1000 / totalEntries) / 10; + if (rate > oldRate) { + ImporterWebsocket.progressUpdated({ rate }); + oldRate = rate; + } + } catch (e) { + console.error(e); } - this.updateRecord({ 'count.channels': tempGroups.length + tempChannels.length + tempDMs.length + tempMpims.length + data.length }); - this.addCountToTotal(data.length); - return data; }; try { + // we need to iterate the zip file twice so that all channels are loaded before the messages + zip.forEach((entry) => { try { - if (entry.entryName.includes('__MACOSX') || entry.entryName.includes('.DS_Store')) { - return this.logger.debug(`Ignoring the file: ${ entry.entryName }`); - } - if (entry.entryName === 'channels.json') { - tempChannels = prepareChannelsFile(entry, 'channels'); - return; + channelCount += this.prepareChannelsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'groups.json') { - tempGroups = prepareChannelsFile(entry, 'groups'); - return; + channelCount += this.prepareGroupsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'mpims.json') { - tempMpims = prepareChannelsFile(entry, 'mpims'); - return; + channelCount += this.prepareMpimpsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'dms.json') { - tempDMs = prepareChannelsFile(entry, 'DMs', false); - return; + channelCount += this.prepareDMsFile(entry); + this.updateRecord({ 'count.channels': channelCount }); + return increaseProgress(); } if (entry.entryName === 'users.json') { - super.updateProgress(ProgressStep.PREPARING_USERS); - tempUsers = JSON.parse(entry.getData().toString()); - - tempUsers.forEach((user) => { - if (user.is_bot) { - this.bots[user.profile.bot_id] = user; - } - }); - - this.logger.debug(`loaded ${ tempUsers.length } users.`); + this.prepareUsersFile(entry); + return increaseProgress(); + } + } catch (e) { + this.logger.error(e); + } + }); - // Insert the users record - if (Base.getBSONSize(tempUsers) > Base.getMaxBSONSize()) { - const tmp = Base.getBSONSafeArraysFromAnArray(tempUsers); - Object.keys(tmp).forEach((i) => { - const splitUsers = tmp[i]; - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', name: `users/${ i }`, users: splitUsers, i }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', name: 'users', users: tempUsers }); - } + const missedTypes = {}; + // If we have no slack message yet, then we can insert them instead of upserting + this._useUpsert = !Messages.findOne({ _id: /slack\-.*/ }); - this.updateRecord({ 'count.users': tempUsers.length }); - this.addCountToTotal(tempUsers.length); + zip.forEach((entry) => { + try { + if (entry.entryName.includes('__MACOSX') || entry.entryName.includes('.DS_Store')) { + count++; + return this.logger.debug(`Ignoring the file: ${ entry.entryName }`); + } - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'bots', bots: this.bots }); + if (['channels.json', 'groups.json', 'mpims.json', 'dms.json', 'users.json'].includes(entry.entryName)) { return; } - if (!entry.isDirectory && entry.entryName.indexOf('/') > -1) { + if (!entry.isDirectory && entry.entryName.includes('/')) { const item = entry.entryName.split('/'); const channel = item[0]; @@ -153,14 +271,12 @@ export class SlackImporter extends Base { this.updateRecord({ messagesstatus: `${ channel }/${ date }` }); this.addCountToTotal(tempMessages.length); - if (Base.getBSONSize(tempMessages) > Base.getMaxBSONSize()) { - const tmp = Base.getBSONSafeArraysFromAnArray(tempMessages); - Object.keys(tmp).forEach((i) => { - const splitMsg = tmp[i]; - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channel }/${ date }.${ i }`, messages: splitMsg, channel, date, i }); - }); - } else { - this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channel }/${ date }`, messages: tempMessages, channel, date }); + const slackChannelId = ImportData.findChannelImportIdByNameOrImportId(channel); + + if (slackChannelId) { + for (const message of tempMessages) { + this.prepareMessageObject(message, missedTypes, slackChannelId); + } } } catch (error) { this.logger.warn(`${ entry.entryName } is not a valid JSON file! Unable to import it.`); @@ -170,17 +286,12 @@ export class SlackImporter extends Base { this.logger.error(e); } - try { - count++; - const rate = Math.floor(count * 1000 / totalEntries) / 10; - if (rate > oldRate) { - ImporterWebsocket.progressUpdated({ rate }); - oldRate = rate; - } - } catch (e) { - console.error(e); - } + increaseProgress(); }); + + if (!_.isEmpty(missedTypes)) { + console.log('Missed import types:', missedTypes); + } } catch (e) { this.logger.error(e); throw e; @@ -188,271 +299,99 @@ export class SlackImporter extends Base { ImporterWebsocket.progressUpdated({ rate: 100 }); this.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); - const roomCount = tempChannels.length + tempGroups.length + tempDMs.length + tempMpims.length; - - if ([tempUsers.length, roomCount, messagesCount].some((e) => e === 0)) { - this.logger.warn(`Loaded ${ tempUsers.length } users, ${ tempChannels.length } channels, ${ tempGroups.length } groups, ${ tempDMs.length } DMs, ${ tempMpims.length } multi party IMs and ${ messagesCount } messages`); - super.updateProgress(ProgressStep.ERROR); - return this.getProgress(); - } - - const selectionUsers = (() => { - if (tempUsers.length <= 500) { - return tempUsers.map((user) => new SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); - } - - return [ - new SelectionUser('users', 'Regular Users', '', false, false, true), - new SelectionUser('bot_users', 'Bot Users', '', false, true, false), - new SelectionUser('deleted_users', 'Deleted Users', '', true, false, true), - ]; - })(); - - const selectionChannels = (() => { - if (roomCount <= 500) { - return tempChannels.map((channel) => new SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); - } - - return [ - new SelectionChannel('channels', 'Regular Channels', false, true, false), - new SelectionChannel('archived_channels', 'Archived Channels', true, true, false), - ]; - })(); - - const selectionGroups = (() => { - if (roomCount <= 500) { - return tempGroups.map((channel) => new SelectionChannel(channel.id, channel.name, channel.is_archived, true, true)); - } - - return [ - new SelectionChannel('groups', 'Regular Groups', false, true, true), - new SelectionChannel('archived_groups', 'Archived Groups', true, true, true), - ]; - })(); - - const selectionMpims = [ - new SelectionChannel('mpims', 'Multi Party DMs', false, true, true), - new SelectionChannel('archived_mimps', 'Archived Multi Party DMs', true, true, true), - ]; - - const selectionMessages = this.importRecord.count.messages; - super.updateProgress(ProgressStep.USER_SELECTION); - - return new Selection(this.name, selectionUsers, selectionMpims.concat(selectionChannels).concat(selectionGroups), selectionMessages); } - performUserImport(user, startedByUserId) { - if (user.is_bot) { - this._saveUserIdReference(user.id, 'rocket.cat', user.name, 'rocket.cat'); - } - - if (!user.do_import) { - this.addCountCompleted(1); - return; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantUser = Users.findOneByEmailAddress(user.profile.email) || Users.findOneByUsernameIgnoringCase(user.name); - if (existantUser) { - user.rocketId = existantUser._id; - Users.update({ _id: user.rocketId }, { $addToSet: { importIds: user.id } }); - this._saveUserIdReference(user.id, existantUser._id, user.name, existantUser.username); - } else { - const userId = user.profile.email ? Accounts.createUser({ email: user.profile.email, password: Date.now() + user.name + user.profile.email.toUpperCase() }) : Accounts.createUser({ username: user.name, password: Date.now() + user.name, joinDefaultChannelsSilenced: true }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', user.name, { joinDefaultChannelsSilenced: true }); - - const url = user.profile.image_original || user.profile.image_512; - if (url) { - try { - Users.update({ _id: userId }, { $set: { _pendingAvatarUrl: url } }); - } catch (error) { - this.logger.warn(`Failed to set ${ user.name }'s avatar from url ${ url }`); - console.log(`Failed to set ${ user.name }'s avatar from url ${ url }`); - } - } - - // Slack's is -18000 which translates to Rocket.Chat's after dividing by 3600 - if (user.tz_offset) { - Meteor.call('userSetUtcOffset', user.tz_offset / 3600); - } - }); - - Users.update({ _id: userId }, { $addToSet: { importIds: user.id } }); - - if (user.profile.real_name) { - Users.setName(userId, user.profile.real_name); - } - - // Deleted users are 'inactive' users in Rocket.Chat - if (user.deleted) { - Meteor.call('setUserActiveStatus', userId, false); - } - - user.rocketId = userId; - this._saveUserIdReference(user.id, userId, user.name, user.name); - } - - this.addCountCompleted(1); - }); - } - - parseMentions(message) { + parseMentions(newMessage) { const mentionsParser = new MentionsParser({ pattern: () => settings.get('UTF8_Names_Validation'), useRealName: () => settings.get('UI_Use_Real_Name'), me: () => 'me', }); - if (!message.mentions) { - message.mentions = []; - } - const users = mentionsParser.getUserMentions(message.msg); - users.forEach((user_id, index, arr) => { - const user = user_id.slice(1, user_id.length); - try { - if (user === 'all' || user === 'here') { - arr[index] = user; - } else { - arr[index] = Users.findOneByUsernameIgnoringCase(user); - } - } catch (e) { - this.logger.warn(`Failed to import user mention with name: ${ user }`); + const users = mentionsParser.getUserMentions(newMessage.msg).filter((u) => u).map((uid) => this._replaceSlackUserId(uid.slice(1, uid.length))); + if (users.length) { + if (!newMessage.mentions) { + newMessage.mentions = []; } - }); - - const filteredUsers = users.filter((u) => u); - message.mentions.push(...filteredUsers); - - if (!message.channels) { - message.channels = []; + newMessage.mentions.push(...users); } - const channels = mentionsParser.getChannelMentions(message.msg); - channels.forEach((channel_name, index, arr) => { - const chan = channel_name.slice(1, channel_name.length); - try { - const slackChannel = this.getSlackChannelFromName(chan); - arr[index] = Rooms.findOneById(slackChannel.rocketId); - arr[index].dname = chan; // Have to store name to display so parser can match it - } catch (e) { - this.logger.warn(`Failed to import channel mention with name: ${ chan }`); - } - }); - const filteredChannels = channels.filter((c) => c); - message.channels.push(...filteredChannels); + const channels = mentionsParser.getChannelMentions(newMessage.msg).filter((c) => c).map((name) => name.slice(1, name.length)); + if (channels.length) { + if (!newMessage.channels) { + newMessage.channels = []; + } + newMessage.channels.push(...channels); + } } - processMessageSubType(message, room, msgDataDefaults, missedTypes) { + processMessageSubType(message, slackChannelId, newMessage, missedTypes) { const ignoreTypes = { bot_add: true, file_comment: true, file_mention: true }; - let rocketUser = this.getRocketUserFromUserId(message.user); - const useRocketCat = !rocketUser; - - if (useRocketCat) { - rocketUser = this.getRocketUserFromUserId('rocket.cat'); - } - - if (!rocketUser) { - return; - } - switch (message.subtype) { case 'channel_join': case 'group_join': - if (!useRocketCat) { - Messages.createUserJoinWithRoomIdAndUser(room._id, rocketUser, msgDataDefaults); - } - break; + newMessage.t = 'uj'; + newMessage.groupable = false; + return true; case 'channel_leave': case 'group_leave': - if (!useRocketCat) { - Messages.createUserLeaveWithRoomIdAndUser(room._id, rocketUser, msgDataDefaults); - } - break; - case 'me_message': { - const msgObj = { - ...msgDataDefaults, - msg: `_${ this.convertSlackMessageToRocketChat(message.text) }_`, - }; - this.parseMentions(msgObj); - insertMessage(rocketUser, msgObj, room, this._anyExistingSlackMessage); - break; - } - case 'bot_message': - case 'slackbot_response': { - const botUser = this.getRocketUserFromUserId('rocket.cat'); - const botUsername = this.bots[message.bot_id] ? this.bots[message.bot_id].name : message.username; - const msgObj = { - ...msgDataDefaults, - msg: this.convertSlackMessageToRocketChat(message.text), - rid: room._id, - bot: true, - attachments: this.convertMessageAttachments(message.attachments), - username: botUsername || undefined, - }; - - if (message.edited) { - msgObj.editedAt = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000); - const editedBy = this.getRocketUserFromUserId(message.edited.user); - if (editedBy) { - msgObj.editedBy = { - _id: editedBy._id, - username: editedBy.username, - }; - } - } - - if (message.icons) { - msgObj.emoji = message.icons.emoji; - } - this.parseMentions(msgObj); - insertMessage(botUser, msgObj, room, this._anyExistingSlackMessage); - break; - } - + newMessage.t = 'ul'; + newMessage.groupable = false; + return true; case 'channel_purpose': case 'group_purpose': - Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_description', room._id, message.purpose, rocketUser, msgDataDefaults); - break; + newMessage.t = 'room_changed_description'; + newMessage.groupable = false; + newMessage.msg = message.purpose; + return true; case 'channel_topic': case 'group_topic': - Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, message.topic, rocketUser, msgDataDefaults); - break; + newMessage.t = 'room_changed_topic'; + newMessage.groupable = false; + newMessage.msg = message.topic; + return true; case 'channel_name': case 'group_name': - Messages.createRoomRenamedWithRoomIdRoomNameAndUser(room._id, message.name, rocketUser, msgDataDefaults); - break; + newMessage.t = 'r'; + newMessage.msg = message.name; + newMessage.groupable = false; + return true; case 'pinned_item': if (message.attachments) { - const msgObj = { - ...msgDataDefaults, - attachments: [{ - text: this.convertSlackMessageToRocketChat(message.attachments[0].text), - author_name: message.attachments[0].author_subname, - author_icon: getUserAvatarURL(message.attachments[0].author_subname), - }], - }; - - Messages.createWithTypeRoomIdMessageAndUser('message_pinned', room._id, '', rocketUser, msgObj); - } else { - // TODO: make this better - this.logger.debug('Pinned item with no attachment, needs work.'); - // Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUserFromUserId(message.user), msgDataDefaults + if (!newMessage.attachments) { + newMessage.attachments = []; + } + newMessage.attachments.push({ + text: this.convertSlackMessageToRocketChat(message.attachments[0].text), + author_name: message.attachments[0].author_subname, + author_icon: getUserAvatarURL(message.attachments[0].author_subname), + }); + newMessage.t = 'message_pinned'; } break; case 'file_share': - if (message.file && message.file.url_private_download !== undefined) { - const details = { - message_id: `slack-${ message.ts.replace(/\./g, '-') }`, - name: message.file.name, - size: message.file.size, - type: message.file.mimetype, - rid: room._id, + if (message.file?.url_private_download) { + const fileId = this.makeSlackMessageId(slackChannelId, message.ts, 'share'); + const fileMessage = { + _id: fileId, + rid: newMessage.rid, + ts: newMessage.ts, + msg: message.file.url_private_download || '', + _importFile: this.convertSlackFileToPendingFile(message.file), + u: { + _id: newMessage.u._id, + }, }; - this.uploadFile(details, message.file.url_private_download, rocketUser, room, new Date(parseInt(message.ts.split('.')[0]) * 1000)); + + if (message.thread_ts && (message.thread_ts !== message.ts)) { + fileMessage.tmid = this.makeSlackMessageId(slackChannelId, message.thread_ts); + } + + this.converter.addMessage(fileMessage, this._useUpsert); } break; + default: if (!missedTypes[message.subtype] && !ignoreTypes[message.subtype]) { missedTypes[message.subtype] = message; @@ -461,690 +400,154 @@ export class SlackImporter extends Base { } } - performMessageImport(message, room, missedTypes, slackChannel) { - const msgDataDefaults = { - _id: `slack-${ slackChannel.id }-${ message.ts.replace(/\./g, '-') }`, + makeSlackMessageId(channelId, ts, fileIndex = undefined) { + const base = `slack-${ channelId }-${ ts.replace(/\./g, '-') }`; + + if (fileIndex) { + return `${ base }-file${ fileIndex }`; + } + + return base; + } + + prepareMessageObject(message, missedTypes, slackChannelId) { + const id = this.makeSlackMessageId(slackChannelId, message.ts); + const newMessage = { + _id: id, + rid: slackChannelId, ts: new Date(parseInt(message.ts.split('.')[0]) * 1000), + u: { + _id: this._replaceSlackUserId(message.user), + }, }; // Process the reactions if (message.reactions && message.reactions.length > 0) { - msgDataDefaults.reactions = {}; + newMessage.reactions = new Map(); message.reactions.forEach((reaction) => { - reaction.name = `:${ reaction.name }:`; - msgDataDefaults.reactions[reaction.name] = { usernames: [] }; - - if (reaction.users) { - reaction.users.forEach((u) => { - const rcUser = this.getRocketUserFromUserId(u); - if (!rcUser) { return; } - - msgDataDefaults.reactions[reaction.name].usernames.push(rcUser.username); + const name = `:${ reaction.name }:`; + if (reaction.users && reaction.users.length) { + newMessage.reactions.set(name, { + name, + users: this._replaceSlackUserIds(reaction.users), }); } - - if (msgDataDefaults.reactions[reaction.name].usernames.length === 0) { - delete msgDataDefaults.reactions[reaction.name]; - } }); } if (message.type === 'message') { if (message.files) { - const fileUser = this.getRocketUserFromUserId(message.user); let fileIndex = 0; - message.files.forEach((file) => { fileIndex++; - const msgObj = { - _id: `slack-${ slackChannel.id }-${ message.ts.replace(/\./g, '-') }-file${ fileIndex }`, - ts: msgDataDefaults.ts, + + const fileId = this.makeSlackMessageId(slackChannelId, message.ts, fileIndex); + const fileMessage = { + _id: fileId, + rid: slackChannelId, + ts: newMessage.ts, msg: file.url_private_download || '', _importFile: this.convertSlackFileToPendingFile(file), - }; - if (message.thread_ts && (message.thread_ts !== message.ts)) { - msgObj.tmid = `slack-${ slackChannel.id }-${ message.thread_ts.replace(/\./g, '-') }`; - } - try { - insertMessage(fileUser, msgObj, room, this._anyExistingSlackMessage); - } catch (e) { - this.logger.warn(`Failed to import the message file: ${ msgDataDefaults._id }-${ fileIndex }`); - this.logger.error(e); - } - }); - } - - if (message.subtype && (message.subtype !== 'thread_broadcast')) { - this.processMessageSubType(message, room, msgDataDefaults, missedTypes); - } else { - const user = this.getRocketUserFromUserId(message.user); - if (user) { - const msgObj = { - ...msgDataDefaults, - msg: this.convertSlackMessageToRocketChat(message.text), - rid: room._id, - attachments: this.convertMessageAttachments(message.attachments), u: { - _id: user._id, - username: user.username, + _id: this._replaceSlackUserId(message.user), }, }; - if (message.thread_ts) { - if (message.thread_ts === message.ts) { - if (message.reply_users) { - msgObj.replies = []; - message.reply_users.forEach(function(item) { - msgObj.replies.push(item); - }); - } else if (message.replies) { - msgObj.replies = []; - message.replies.forEach(function(item) { - msgObj.replies.push(item.user); - }); - } else { - this.logger.warn(`Failed to import the parent comment, message: ${ msgDataDefaults._id }. Missing replies/reply_users field`); - } - - msgObj.tcount = message.reply_count; - msgObj.tlm = new Date(parseInt(message.latest_reply.split('.')[0]) * 1000); - } else { - msgObj.tmid = `slack-${ slackChannel.id }-${ message.thread_ts.replace(/\./g, '-') }`; - } - } - - if (message.edited) { - msgObj.editedAt = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000); - const editedBy = this.getRocketUserFromUserId(message.edited.user); - if (editedBy) { - msgObj.editedBy = { - _id: editedBy._id, - username: editedBy.username, - }; - } - } - - this.parseMentions(msgObj); - try { - insertMessage(this.getRocketUserFromUserId(message.user), msgObj, room, this._anyExistingSlackMessage); - } catch (e) { - this.logger.warn(`Failed to import the message: ${ msgDataDefaults._id }`); - this.logger.error(e); - } - } - } - } - - this.addCountCompleted(1); - } - - _saveUserIdReference(slackId, rocketId, slackUsername, rocketUsername) { - this._userIdReference[slackId] = rocketId; - - this.userTags.push({ - slack: `<@${ slackId }>`, - slackLong: `<@${ slackId }|${ slackUsername }>`, - rocket: `@${ rocketUsername }`, - }); - } - - _getUserRocketId(slackId) { - if (!this._userIdReference) { - return; - } - - return this._userIdReference[slackId]; - } - - _importUsers(startedByUserId) { - this._userIdReference = {}; - - super.updateProgress(ProgressStep.IMPORTING_USERS); - for (const list of this.userLists) { - list.users.forEach((user) => this.performUserImport(user, startedByUserId)); - this.collection.update({ _id: list._id }, { $set: { users: list.users } }); - } - } - - _importChannels(startedByUserId, channelNames) { - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - - for (const list of this.channelsLists) { - list.channels.forEach((channel) => { - if (!channel.do_import) { - this.addCountCompleted(1); - return; - } - - if (channelNames.includes(channel.name)) { - this.logger.warn(`Duplicated channel name will be skipped: ${ channel.name }`); - return; - } - channelNames.push(channel.name); - - Meteor.runAsUser(startedByUserId, () => { - const existingRoom = this._findExistingRoom(channel.name); - - if (existingRoom || channel.is_general) { - if (channel.is_general && existingRoom && channel.name !== existingRoom.name) { - Meteor.call('saveRoomSettings', 'GENERAL', 'roomName', channel.name); - } - - channel.rocketId = channel.is_general ? 'GENERAL' : existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - const users = this._getChannelUserList(channel); - const userId = this.getImportedRocketUserIdFromSlackUserId(channel.creator) || startedByUserId; - - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createChannel', channel.name, users); - channel.rocketId = returned.rid; - }); - - this._updateImportedChannelTopicAndDescription(channel); + if (message.thread_ts && (message.thread_ts !== message.ts)) { + fileMessage.tmid = this.makeSlackMessageId(slackChannelId, message.thread_ts); } - this.addCountCompleted(1); + this.converter.addMessage(fileMessage, this._useUpsert); }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _findExistingRoom(name) { - const existingRoom = Rooms.findOneByName(name); - - // If no room with that name is found, try applying name rules and searching again - if (!existingRoom) { - const newName = getValidRoomName(name, null, { allowDuplicates: true }); - - if (newName !== name) { - return Rooms.findOneByName(newName); } - } - return existingRoom; - } - - _getChannelUserList(channel, returnObject = false, includeCreator = false) { - return channel.members.reduce((ret, member) => { - if (includeCreator || member !== channel.creator) { - const user = this.getRocketUserFromUserId(member); - // Don't add bots to the room's member list; Since they are all replaced with rocket.cat, it could cause duplicated subscriptions - if (user && user.username && user._id !== 'rocket.cat') { - if (returnObject) { - ret.push(user); - } else { - ret.push(user.username); - } - } - } - return ret; - }, []); - } + const regularTypes = [ + 'me_message', + 'thread_broadcast', + ]; - _importPrivateGroupList(startedByUserId, listList, channelNames) { - for (const list of listList) { - list.channels.forEach((channel) => { - if (!channel.do_import) { - this.addCountCompleted(1); - return; - } + const isBotMessage = message.subtype && ['bot_message', 'slackbot_response'].includes(message.subtype); - if (channelNames.includes(channel.name)) { - this.logger.warn(`Duplicated group name will be skipped: ${ channel.name }`); - return; + if (message.subtype && !regularTypes.includes(message.subtype) && !isBotMessage) { + if (this.processMessageSubType(message, slackChannelId, newMessage, missedTypes)) { + this.converter.addMessage(newMessage, this._useUpsert); } + } else { + const text = this.convertSlackMessageToRocketChat(message.text); - channelNames.push(channel.name); - - Meteor.runAsUser(startedByUserId, () => { - const existingRoom = this._findExistingRoom(channel.name); - - if (existingRoom) { - channel.rocketId = existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - const users = this._getChannelUserList(channel); - - const userId = this.getImportedRocketUserIdFromSlackUserId(channel.creator) || startedByUserId; - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createPrivateGroup', channel.name, users); - channel.rocketId = returned.rid; - }); - - this._updateImportedChannelTopicAndDescription(channel); - } - - this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _importGroups(startedByUserId, channelNames) { - this._importPrivateGroupList(startedByUserId, this.groupsLists, channelNames); - } - - _updateImportedChannelTopicAndDescription(slackChannel) { - // @TODO implement model specific function - const roomUpdate = { - ts: new Date(slackChannel.created * 1000), - }; - - if (!_.isEmpty(slackChannel.topic && slackChannel.topic.value)) { - roomUpdate.topic = slackChannel.topic.value; - } - - if (!_.isEmpty(slackChannel.purpose && slackChannel.purpose.value)) { - roomUpdate.description = slackChannel.purpose.value; - } - - Rooms.update({ _id: slackChannel.rocketId }, { $set: roomUpdate, $addToSet: { importIds: slackChannel.id } }); - } - - _importMpims(startedByUserId, channelNames) { - const maxUsers = settings.get('DirectMesssage_maxUsers') || 1; - - - for (const list of this.mpimsLists) { - list.channels.forEach((channel) => { - if (!channel.do_import) { - this.addCountCompleted(1); - return; + if (isBotMessage) { + newMessage.bot = true; } - if (channelNames.includes(channel.name)) { - this.logger.warn(`Duplicated multi party IM name will be skipped: ${ channel.name }`); - return; + if (message.subtype === 'me_message') { + newMessage.msg = `_${ text }_`; + } else { + newMessage.msg = text; } - channelNames.push(channel.name); - - Meteor.runAsUser(startedByUserId, () => { - const users = this._getChannelUserList(channel, true, true); - const existingRoom = Rooms.findOneDirectRoomContainingAllUserIDs(users, { fields: { _id: 1 } }); + if (message.thread_ts) { + if (message.thread_ts === message.ts) { + if (message.reply_users) { + const replies = new Set(); + message.reply_users.forEach((item) => { + replies.add(this._replaceSlackUserId(item)); + }); - if (existingRoom) { - channel.rocketId = existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - const userId = this.getImportedRocketUserIdFromSlackUserId(channel.creator) || startedByUserId; - Meteor.runAsUser(userId, () => { - // If there are too many users for a direct room, then create a private group instead - if (users.length > maxUsers) { - const usernames = users.map((user) => user.username); - const group = Meteor.call('createPrivateGroup', channel.name, usernames); - channel.rocketId = group.rid; - return; + if (replies.length) { + newMessage.replies = Array.from(replies); } + } else if (message.replies) { + const replies = new Set(); + message.repĺies.forEach((item) => { + replies.add(this._replaceSlackUserId(item.user)); + }); - const newRoom = createDirectRoom(users); - channel.rocketId = newRoom._id; - }); - - this._updateImportedChannelTopicAndDescription(channel); - } - - this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _importDMs(startedByUserId, channelNames) { - for (const list of this.dmsLists) { - list.channels.forEach((channel) => { - if (channelNames.includes(channel.id)) { - this.logger.warn(`Duplicated DM id will be skipped (DMs): ${ channel.id }`); - return; - } - channelNames.push(channel.id); - - if (!channel.members || channel.members.length !== 2) { - this.addCountCompleted(1); - return; - } - - Meteor.runAsUser(startedByUserId, () => { - const user1 = this.getRocketUserFromUserId(channel.members[0]); - const user2 = this.getRocketUserFromUserId(channel.members[1]); - - const existingRoom = Rooms.findOneDirectRoomContainingAllUserIDs([user1, user2], { fields: { _id: 1 } }); - - if (existingRoom) { - channel.rocketId = existingRoom._id; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } else { - if (!user1) { - this.logger.error(`DM creation: User not found for id ${ channel.members[0] } and channel id ${ channel.id }`); - return; - } - - if (!user2) { - this.logger.error(`DM creation: User not found for id ${ channel.members[1] } and channel id ${ channel.id }`); - return; - } - - const roomInfo = Meteor.runAsUser(user1._id, () => Meteor.call('createDirectMessage', user2.username)); - channel.rocketId = roomInfo.rid; - Rooms.update({ _id: channel.rocketId }, { $addToSet: { importIds: channel.id } }); - } - - this.addCountCompleted(1); - }); - }); - - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - } - - _importMessages(startedByUserId, channelNames) { - const missedTypes = {}; - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - for (const channel of channelNames) { - if (!channel) { - continue; - } - - const slackChannel = this.getSlackChannelFromName(channel); - - const room = Rooms.findOneById(slackChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); - if (!room) { - this.logger.error(`ROOM not found: ${ channel }`); - continue; - } - const messagePacks = this.collection.find({ import: this.importRecord._id, type: 'messages', channel }); - - Meteor.runAsUser(startedByUserId, () => { - messagePacks.forEach((pack) => { - const packId = pack.i ? `${ pack.date }.${ pack.i }` : pack.date; - - this.updateRecord({ messagesstatus: `${ channel }/${ packId } (${ pack.messages.length })` }); - pack.messages.forEach((message) => { - try { - return this.performMessageImport(message, room, missedTypes, slackChannel); - } catch (e) { - this.logger.warn(`Failed to import message with timestamp ${ String(message.ts) } to room ${ room._id }`); - this.logger.debug(e); + if (replies.length) { + newMessage.replies = Array.from(replies); + } + } else { + this.logger.warn(`Failed to import the parent comment, message: ${ newMessage._id }. Missing replies/reply_users field`); } - }); - }); - }); - } - - if (!_.isEmpty(missedTypes)) { - console.log('Missed import types:', missedTypes); - } - } - _applyUserSelection(importSelection) { - if (importSelection.users.length === 3 && importSelection.users[0].user_id === 'users') { - const regularUsers = importSelection.users[0].do_import; - const botUsers = importSelection.users[1].do_import; - const deletedUsers = importSelection.users[2].do_import; - - for (const list of this.userLists) { - Object.keys(list.users).forEach((k) => { - const u = list.users[k]; - - if (u.is_bot) { - u.do_import = botUsers; - } else if (u.deleted) { - u.do_import = deletedUsers; + newMessage.tcount = message.reply_count; + newMessage.tlm = new Date(parseInt(message.latest_reply.split('.')[0]) * 1000); } else { - u.do_import = regularUsers; - } - }); - - this.collection.update({ _id: list._id }, { $set: { users: list.users } }); - } - } - - Object.keys(importSelection.users).forEach((key) => { - const user = importSelection.users[key]; - - for (const list of this.userLists) { - Object.keys(list.users).forEach((k) => { - const u = list.users[k]; - if (u.id === user.user_id) { - u.do_import = user.do_import; - } - }); - - this.collection.update({ _id: list._id }, { $set: { users: list.users } }); - } - }); - } - - _applyChannelSelection(importSelection) { - const iterateChannelList = (listList, channel_id, do_import) => { - for (const list of listList) { - for (const c of list.channels) { - if (!c) { - continue; - } - - if (channel_id === '*') { - if (!c.archived) { - c.do_import = do_import; - } - } else if (channel_id === '*/archived') { - if (c.archived) { - c.do_import = do_import; - } - } else if (c.id === channel_id) { - c.do_import = do_import; + newMessage.tmid = this.makeSlackMessageId(slackChannelId, message.thread_ts); } } - this.collection.update({ _id: list._id }, { $set: { channels: list.channels } }); - } - }; - - Object.keys(importSelection.channels).forEach((key) => { - const channel = importSelection.channels[key]; - - switch (channel.channel_id) { - case 'channels': - iterateChannelList(this.channelsLists, '*', channel.do_import); - break; - case 'archived_channels': - iterateChannelList(this.channelsLists, '*/archived', channel.do_import); - break; - case 'groups': - iterateChannelList(this.groupsLists, '*', channel.do_import); - break; - case 'archived_groups': - iterateChannelList(this.groupsLists, '*/archived', channel.do_import); - break; - case 'mpims': - iterateChannelList(this.mpimsLists, '*', channel.do_import); - break; - case 'archived_mpims': - iterateChannelList(this.mpimsLists, '*/archived', channel.do_import); - break; - default: - iterateChannelList(this.channelsLists, channel.channel_id, channel.do_import); - iterateChannelList(this.groupsLists, channel.channel_id, channel.do_import); - iterateChannelList(this.mpimsLists, channel.channel_id, channel.do_import); - break; - } - }); - } - - startImport(importSelection) { - const bots = this.collection.findOne({ import: this.importRecord._id, type: 'bots' }); - if (bots) { - this.bots = bots.bots || {}; - } else { - this.bots = {}; - } - - this.userLists = RawImports.find({ import: this.importRecord._id, type: 'users' }).fetch(); - this.channelsLists = RawImports.find({ import: this.importRecord._id, type: 'channels' }).fetch(); - this.groupsLists = RawImports.find({ import: this.importRecord._id, type: 'groups' }).fetch(); - this.dmsLists = RawImports.find({ import: this.importRecord._id, type: 'DMs' }).fetch(); - this.mpimsLists = RawImports.find({ import: this.importRecord._id, type: 'mpims' }).fetch(); - - this._userDataCache = {}; - this._anyExistingSlackMessage = Boolean(Messages.findOne({ _id: /slack\-.*/ })); - this.reloadCount(); - - super.startImport(importSelection); - const start = Date.now(); - - this._applyUserSelection(importSelection); - this._applyChannelSelection(importSelection); - - const channelNames = []; - - const startedByUserId = Meteor.userId(); - Meteor.defer(() => { - try { - this._importUsers(startedByUserId); - - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - this._importChannels(startedByUserId, channelNames); - this._importGroups(startedByUserId, channelNames); - this._importMpims(startedByUserId, channelNames); - - this._importDMs(startedByUserId, channelNames); - - this._importMessages(startedByUserId, channelNames); - - super.updateProgress(ProgressStep.FINISHING); - - try { - this._archiveChannelsAsNeeded(startedByUserId, this.channelsLists); - this._archiveChannelsAsNeeded(startedByUserId, this.groupsLists); - this._archiveChannelsAsNeeded(startedByUserId, this.mpimsLists); - - this._updateRoomsLastMessage(this.channelsLists); - this._updateRoomsLastMessage(this.groupsLists); - this._updateRoomsLastMessage(this.mpimsLists); - this._updateRoomsLastMessage(this.dmsLists); - } catch (e) { - // If it failed to archive some channel, it's no reason to flag the import as incomplete - // Just report the error but keep the import as successful. - console.error(e); + if (message.edited) { + newMessage.editedAt = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000); + if (message.edited.user) { + newMessage.editedBy = this._replaceSlackUserId(message.edited.user); + } } - super.updateProgress(ProgressStep.DONE); - - this.logger.log(`Import took ${ Date.now() - start } milliseconds.`); - } catch (e) { - this.logger.error(e); - super.updateProgress(ProgressStep.ERROR); - } - - this._userIdReference = {}; - }); - - return this.getProgress(); - } - _archiveChannelsAsNeeded(startedByUserId, listList) { - for (const list of listList) { - list.channels.forEach((channel) => { - if (channel.do_import && channel.is_archived && channel.rocketId) { - Meteor.runAsUser(startedByUserId, function() { - Meteor.call('archiveRoom', channel.rocketId); - }); + if (message.attachments) { + newMessage.attachments = this.convertMessageAttachments(message.attachments); } - }); - } - } - _updateRoomsLastMessage(listList) { - for (const list of listList) { - list.channels.forEach((channel) => { - if (channel.do_import && channel.rocketId) { - Rooms.resetLastMessageById(channel.rocketId); + if (message.icons && message.icons.emoji) { + newMessage.emoji = message.icons.emoji; } - }); - } - } - - getSlackChannelFromName(channelName) { - for (const list of this.channelsLists) { - const channel = list.channels.find((channel) => channel.name === channelName); - if (channel) { - return channel; - } - } - for (const list of this.groupsLists) { - const group = list.channels.find((channel) => channel.name === channelName); - if (group) { - return group; - } - } - - for (const list of this.mpimsLists) { - const group = list.channels.find((channel) => channel.name === channelName); - if (group) { - return group; - } - } - - for (const list of this.dmsLists) { - const dm = list.channels.find((channel) => channel.id === channelName); - if (dm) { - return dm; + this.parseMentions(newMessage); + this.converter.addMessage(newMessage, this._useUpsert); } } } - _getBasicUserData(userId) { - if (this._userDataCache[userId]) { - return this._userDataCache[userId]; - } - - this._userDataCache[userId] = Users.findOneById(userId, { fields: { username: 1, name: 1 } }); - return this._userDataCache[userId]; - } - - getRocketUserFromUserId(userId) { - if (userId === 'rocket.cat' || userId === 'USLACKBOT') { - return this._getBasicUserData('rocket.cat'); - } - - const rocketId = this._getUserRocketId(userId); - if (rocketId) { - return this._getBasicUserData(rocketId); - } - - if (userId in this.bots) { - return this._getBasicUserData('rocket.cat'); - } - } - - getImportedRocketUserIdFromSlackUserId(slackUserId) { - if (slackUserId.toUpperCase() === 'USLACKBOT') { + _replaceSlackUserId(userId) { + if (userId === 'USLACKBOT') { return 'rocket.cat'; } - for (const list of this.userLists) { - for (const user of list.users) { - if (user.id !== slackUserId) { - continue; - } - - if (user.do_import) { - return user.rocketId; - } + return userId; + } - if (user.is_bot) { - return 'rocket.cat'; - } - } - } + _replaceSlackUserIds(members) { + return members.map((userId) => this._replaceSlackUserId(userId)); } convertSlackMessageToRocketChat(message) { @@ -1162,11 +565,8 @@ export class SlackImporter extends Base { message = message.replace(/<(http[s]?:[^>|]*)>/g, '$1'); message = message.replace(/<(http[s]?:[^|]*)\|([^>]*)>/g, '[$2]($1)'); message = message.replace(/<#([^|]*)\|([^>]*)>/g, '#$2'); - - for (const userReplace of Array.from(this.userTags)) { - message = message.replace(userReplace.slack, userReplace.rocket); - message = message.replace(userReplace.slackLong, userReplace.rocket); - } + message = message.replace(/<@([^|]*)\|([^>]*)>/g, '@$1'); + message = message.replace(/<@([^|>]*)>/g, '@$1'); } else { message = ''; } diff --git a/app/importer-slack/server/index.js b/app/importer-slack/server/index.js index 44a1b3bab84c5..d8499b517f006 100644 --- a/app/importer-slack/server/index.js +++ b/app/importer-slack/server/index.js @@ -1 +1,5 @@ -import './adder'; +import { SlackImporter } from './importer'; +import { Importers } from '../../importer/server'; +import { SlackImporterInfo } from '../lib/info'; + +Importers.add(new SlackImporterInfo(), SlackImporter); diff --git a/app/importer/server/classes/ImportDataConverter.ts b/app/importer/server/classes/ImportDataConverter.ts new file mode 100644 index 0000000000000..7ecdeedf78c6f --- /dev/null +++ b/app/importer/server/classes/ImportDataConverter.ts @@ -0,0 +1,832 @@ +import { Meteor } from 'meteor/meteor'; +import { Accounts } from 'meteor/accounts-base'; +import _ from 'underscore'; + +import { ImportData } from '../models/ImportData'; +import { IImportUser } from '../definitions/IImportUser'; +import { IImportMessage, IImportMessageReaction } from '../definitions/IImportMessage'; +import { IImportChannel } from '../definitions/IImportChannel'; +import { IImportUserRecord, IImportChannelRecord, IImportMessageRecord } from '../definitions/IImportRecord'; +import { Users, Rooms, Subscriptions } from '../../../models/server'; +import { generateUsernameSuggestion, insertMessage } from '../../../lib/server'; +import { setUserActiveStatus } from '../../../lib/server/functions/setUserActiveStatus'; +import { IUser } from '../../../../definition/IUser'; + +type IRoom = Record; +type IMessage = Record; +type IUserIdentification = { + _id: string; + username: string | undefined; +}; +type IMentionedUser = { + _id: string; + username: string; + name?: string; +}; +type IMentionedChannel = { + _id: string; + name: string; +}; + +type IMessageReaction = { + name: string; + usernames: Array; +}; + +type IMessageReactions = Record; + +interface IConversionCallbacks { + beforeImportFn?: { + (data: IImportUser | IImportChannel | IImportMessage, type: string): boolean; + }; + afterImportFn?: { + (data: IImportUser | IImportChannel | IImportMessage, type: string): void; + }; +} + +const guessNameFromUsername = (username: string): string => + username + .replace(/\W/g, ' ') + .replace(/\s(.)/g, (u) => u.toUpperCase()) + .replace(/^(.)/, (u) => u.toLowerCase()) + .replace(/^\w/, (u) => u.toUpperCase()); + +export class ImportDataConverter { + private _userCache: Map; + + // display name uses a different cache because it's only used on mentions so we don't need to load it every time we load an user + private _userDisplayNameCache: Map; + + private _roomCache: Map; + + private _roomNameCache: Map; + + constructor() { + this._userCache = new Map(); + this._userDisplayNameCache = new Map(); + this._roomCache = new Map(); + this._roomNameCache = new Map(); + } + + addUserToCache(importId: string, _id: string, username: string | undefined): IUserIdentification { + const cache = { + _id, + username, + }; + + this._userCache.set(importId, cache); + return cache; + } + + addUserDisplayNameToCache(importId: string, name: string): string { + this._userDisplayNameCache.set(importId, name); + return name; + } + + addRoomToCache(importId: string, rid: string): string { + this._roomCache.set(importId, rid); + return rid; + } + + addRoomNameToCache(importId: string, name: string): string { + this._roomNameCache.set(importId, name); + return name; + } + + addUserDataToCache(userData: IImportUser): void { + if (!userData._id) { + return; + } + if (!userData.importIds.length) { + return; + } + + this.addUserToCache(userData.importIds[0], userData._id, userData.username); + } + + addObject(type: string, data: Record, options: Record = {}): void { + ImportData.model.rawCollection().insert({ + data, + dataType: type, + ...options, + }); + } + + addUser(data: IImportUser): void { + this.addObject('user', data); + } + + addChannel(data: IImportChannel): void { + this.addObject('channel', data); + } + + addMessage(data: IImportMessage, useQuickInsert = false): void { + this.addObject('message', data, { + useQuickInsert: useQuickInsert || undefined, + }); + } + + updateUserId(_id: string, userData: IImportUser): void { + const updateData: Record = { + $set: { + statusText: userData.statusText || undefined, + roles: userData.roles || ['user'], + type: userData.type || 'user', + bio: userData.bio || undefined, + name: userData.name || undefined, + }, + }; + + if (userData.importIds?.length) { + updateData.$addToSet = { + importIds: { + $each: userData.importIds, + }, + }; + } + + Users.update({ _id }, updateData); + } + + updateUser(existingUser: IUser, userData: IImportUser): void { + userData._id = existingUser._id; + + this.updateUserId(userData._id, userData); + + if (userData.importIds.length) { + this.addUserToCache(userData.importIds[0], existingUser._id, existingUser.username); + } + + if (userData.avatarUrl) { + try { + Users.update({ _id: existingUser._id }, { $set: { _pendingAvatarUrl: userData.avatarUrl } }); + } catch (error) { + console.warn(`Failed to set ${ existingUser._id }'s avatar from url ${ userData.avatarUrl }`); + console.error(error); + } + } + } + + insertUser(userData: IImportUser): IUser { + const password = `${ Date.now() }${ userData.name || '' }${ userData.emails.length ? userData.emails[0].toUpperCase() : '' }`; + const userId = userData.emails.length ? Accounts.createUser({ + email: userData.emails[0], + password, + }) : Accounts.createUser({ + username: userData.username, + password, + // @ts-ignore + joinDefaultChannelsSilenced: true, + }); + + userData._id = userId; + const user = Users.findOneById(userId, {}); + + if (user && userData.importIds.length) { + this.addUserToCache(userData.importIds[0], user._id, userData.username); + } + + Meteor.runAsUser(userId, () => { + Meteor.call('setUsername', userData.username, { joinDefaultChannelsSilenced: true }); + if (userData.name) { + Users.setName(userId, userData.name); + } + + this.updateUserId(userId, userData); + + if (userData.utcOffset) { + Users.setUtcOffset(userId, userData.utcOffset); + } + + if (userData.avatarUrl) { + try { + Users.update({ _id: userId }, { $set: { _pendingAvatarUrl: userData.avatarUrl } }); + } catch (error) { + console.warn(`Failed to set ${ userId }'s avatar from url ${ userData.avatarUrl }`); + console.error(error); + } + } + }); + + return user; + } + + convertUsers({ beforeImportFn, afterImportFn }: IConversionCallbacks = {}): void { + const users = ImportData.find({ dataType: 'user' }); + users.forEach(({ data, _id }: IImportUserRecord) => { + try { + if (beforeImportFn && !beforeImportFn(data, 'user')) { + this.skipRecord(_id); + return; + } + + data.emails = data.emails.filter((item) => item); + data.importIds = data.importIds.filter((item) => item); + + if (!data.emails.length && !data.username) { + throw new Error('importer-user-missing-email-and-username'); + } + + let existingUser; + if (data.emails.length) { + existingUser = Users.findOneByEmailAddress(data.emails[0], {}); + } + + if (data.username) { + // If we couldn't find one by their email address, try to find an existing user by their username + if (!existingUser) { + existingUser = Users.findOneByUsernameIgnoringCase(data.username, {}); + } + } else { + data.username = generateUsernameSuggestion({ + name: data.name, + emails: data.emails, + }); + } + + if (existingUser) { + this.updateUser(existingUser, data); + } else { + if (!data.name && data.username) { + data.name = guessNameFromUsername(data.username); + } + + existingUser = this.insertUser(data); + } + + // Deleted users are 'inactive' users in Rocket.Chat + if (data.deleted && existingUser?.active) { + setUserActiveStatus(data._id, false, true); + } + + if (afterImportFn) { + afterImportFn(data, 'user'); + } + } catch (e) { + this.saveError(_id, e); + } + }); + } + + saveNewId(importId: string, newId: string): void { + ImportData.update({ + _id: importId, + }, { + $set: { + id: newId, + }, + }); + } + + saveError(importId: string, error: Error): void { + console.error(error); + ImportData.update({ + _id: importId, + }, { + $push: { + errors: { + message: error.message, + stack: error.stack, + }, + }, + }); + } + + skipRecord(_id: string): void { + ImportData.update({ + _id, + }, { + $set: { + skipped: true, + }, + }); + } + + convertMessageReactions(importedReactions: Record): undefined | IMessageReactions { + const reactions: IMessageReactions = {}; + + for (const name in importedReactions) { + if (!importedReactions.hasOwnProperty(name)) { + continue; + } + const { users } = importedReactions[name]; + + if (!users.length) { + continue; + } + + const reaction: IMessageReaction = { + name, + usernames: [], + }; + + for (const importId of users) { + const username = this.findImportedUsername(importId); + if (username && !reaction.usernames.includes(username)) { + reaction.usernames.push(username); + } + } + + if (reaction.usernames.length) { + reactions[name] = reaction; + } + } + + if (Object.keys(reactions).length > 0) { + return reactions; + } + } + + convertMessageReplies(replies: Array): Array { + const result: Array = []; + for (const importId of replies) { + const userId = this.findImportedUserId(importId); + if (userId && !result.includes(userId)) { + result.push(userId); + } + } + return result; + } + + convertMessageMentions(message: IImportMessage): Array | undefined { + const { mentions } = message; + if (!mentions) { + return undefined; + } + + const result: Array = []; + for (const importId of mentions) { + // eslint-disable-next-line no-extra-parens + if (importId === ('all' as 'string') || importId === 'here') { + result.push({ + _id: importId, + username: importId, + }); + continue; + } + + // Loading the name will also store the remaining data on the cache if it's missing, so this won't run two queries + const name = this.findImportedUserDisplayName(importId); + const data = this.findImportedUser(importId); + + if (!data) { + throw new Error('importer-message-mentioned-user-not-found'); + } + if (!data.username) { + throw new Error('importer-message-mentioned-username-not-found'); + } + + message.msg = message.msg.replace(new RegExp(`\@${ importId }`, 'gi'), `@${ data.username }`); + + result.push({ + _id: data._id, + username: data.username as 'string', + name, + }); + } + return result; + } + + convertMessageChannels(message: IImportMessage): Array | undefined { + const { channels } = message; + if (!channels) { + return; + } + + const result: Array = []; + for (const importId of channels) { + // loading the name will also store the id on the cache if it's missing, so this won't run two queries + const name = this.findImportedRoomName(importId); + const _id = this.findImportedRoomId(importId); + + if (!_id || !name) { + console.warn(`Mentioned room not found: ${ importId }`); + continue; + } + + message.msg = message.msg.replace(new RegExp(`\#${ importId }`, 'gi'), `#${ name }`); + + result.push({ + _id, + name, + }); + } + + return result; + } + + convertMessages({ beforeImportFn, afterImportFn }: IConversionCallbacks = {}): void { + const rids: Array = []; + const messages = ImportData.find({ dataType: 'message' }); + messages.forEach(({ data: m, _id }: IImportMessageRecord) => { + try { + if (beforeImportFn && !beforeImportFn(m, 'message')) { + this.skipRecord(_id); + return; + } + + if (!m.ts || isNaN(m.ts as unknown as number)) { + throw new Error('importer-message-invalid-timestamp'); + } + + const creator = this.findImportedUser(m.u._id); + if (!creator) { + console.warn(`Imported user not found: ${ m.u._id }`); + throw new Error('importer-message-unknown-user'); + } + + const rid = this.findImportedRoomId(m.rid); + if (!rid) { + throw new Error('importer-message-unknown-room'); + } + if (!rids.includes(rid)) { + rids.push(rid); + } + + // Convert the mentions and channels first because these conversions can also modify the msg in the message object + const mentions = m.mentions && this.convertMessageMentions(m); + const channels = m.channels && this.convertMessageChannels(m); + + const msgObj: IMessage = { + rid, + u: { + _id: creator._id, + username: creator.username, + }, + msg: m.msg, + ts: m.ts, + t: m.t || undefined, + groupable: m.groupable, + tmid: m.tmid, + tlm: m.tlm, + tcount: m.tcount, + replies: m.replies && this.convertMessageReplies(m.replies), + editedAt: m.editedAt, + editedBy: m.editedBy && (this.findImportedUser(m.editedBy) || undefined), + mentions, + channels, + _importFile: m._importFile, + url: m.url, + attachments: m.attachments, + bot: m.bot, + emoji: m.emoji, + alias: m.alias, + }; + + if (m._id) { + msgObj._id = m._id; + } + + if (m.reactions) { + msgObj.reactions = this.convertMessageReactions(m.reactions); + } + + try { + insertMessage(creator, msgObj, rid, true); + } catch (e) { + console.warn(`Failed to import message with timestamp ${ String(msgObj.ts) } to room ${ rid }`); + console.error(e); + } + + if (afterImportFn) { + afterImportFn(m, 'message'); + } + } catch (e) { + this.saveError(_id, e); + } + }); + + for (const rid of rids) { + try { + Rooms.resetLastMessageById(rid); + } catch (e) { + console.warn(`Failed to update last message of room ${ rid }`); + console.error(e); + } + } + } + + updateRoom(room: IRoom, roomData: IImportChannel, startedByUserId: string): void { + roomData._id = room._id; + + // eslint-disable-next-line no-extra-parens + if ((roomData._id as string).toUpperCase() === 'GENERAL' && roomData.name !== room.name) { + Meteor.runAsUser(startedByUserId, () => { + Meteor.call('saveRoomSettings', 'GENERAL', 'roomName', roomData.name); + }); + } + + this.updateRoomId(room._id, roomData); + } + + findDMForImportedUsers(...users: Array): IImportChannel | undefined { + const record = ImportData.findDMForImportedUsers(...users); + if (record) { + return record.data; + } + } + + findImportedRoomId(importId: string): string | null { + if (this._roomCache.has(importId)) { + return this._roomCache.get(importId) as string; + } + + const options = { + fields: { + _id: 1, + }, + }; + + const room = Rooms.findOneByImportId(importId, options); + if (room) { + return this.addRoomToCache(importId, room._id); + } + + return null; + } + + findImportedRoomName(importId: string): string | undefined { + if (this._roomNameCache.has(importId)) { + return this._roomNameCache.get(importId) as string; + } + + const options = { + fields: { + _id: 1, + name: 1, + }, + }; + + const room = Rooms.findOneByImportId(importId, options); + if (room) { + if (!this._roomCache.has(importId)) { + this.addRoomToCache(importId, room._id); + } + return this.addRoomNameToCache(importId, room.name); + } + } + + findImportedUser(importId: string): IUserIdentification | null { + const options = { + fields: { + _id: 1, + username: 1, + }, + }; + + if (importId === 'rocket.cat') { + return { + _id: 'rocket.cat', + username: 'rocket.cat', + }; + } + + if (this._userCache.has(importId)) { + return this._userCache.get(importId) as IUserIdentification; + } + + const user = Users.findOneByImportId(importId, options); + if (user) { + return this.addUserToCache(importId, user._id, user.username); + } + + return null; + } + + findImportedUserId(_id: string): string | undefined { + const data = this.findImportedUser(_id); + return data?._id; + } + + findImportedUsername(_id: string): string | undefined { + const data = this.findImportedUser(_id); + return data?.username; + } + + findImportedUserDisplayName(importId: string): string | undefined { + const options = { + fields: { + _id: 1, + name: 1, + username: 1, + }, + }; + + if (this._userDisplayNameCache.has(importId)) { + return this._userDisplayNameCache.get(importId); + } + + const user = importId === 'rocket.cat' ? Users.findOneById('rocket.cat', options) : Users.findOneByImportId(importId, options); + if (user) { + if (!this._userCache.has(importId)) { + this.addUserToCache(importId, user._id, user.username); + } + + return this.addUserDisplayNameToCache(importId, user.name); + } + } + + updateRoomId(_id: string, roomData: IImportChannel): void { + const set = { + ts: roomData.ts, + topic: roomData.topic, + description: roomData.description, + }; + + const roomUpdate: {$set?: Record; $addToSet?: Record} = {}; + + if (Object.keys(set).length > 0) { + roomUpdate.$set = set; + } + + if (roomData.importIds.length) { + roomUpdate.$addToSet = { + importIds: { + $each: roomData.importIds, + }, + }; + } + + if (roomUpdate.$set || roomUpdate.$addToSet) { + Rooms.update({ _id: roomData._id }, roomUpdate); + } + } + + getRoomCreatorId(roomData: IImportChannel, startedByUserId: string): string { + if (roomData.u) { + const creatorId = this.findImportedUserId(roomData.u._id); + if (creatorId) { + return creatorId; + } + + if (roomData.t !== 'd') { + return startedByUserId; + } + + throw new Error('importer-channel-invalid-creator'); + } + + if (roomData.t === 'd') { + for (const member of roomData.users) { + const userId = this.findImportedUserId(member); + if (userId) { + return userId; + } + } + } + + throw new Error('importer-channel-invalid-creator'); + } + + insertRoom(roomData: IImportChannel, startedByUserId: string): void { + // Find the rocketchatId of the user who created this channel + const creatorId = this.getRoomCreatorId(roomData, startedByUserId); + const members = this.convertImportedIdsToUsernames(roomData.users, roomData.t !== 'd' ? creatorId : undefined); + + if (roomData.t === 'd') { + if (members.length < roomData.users.length) { + console.warn('One or more imported users not found: ${ roomData.users }'); + throw new Error('importer-channel-missing-users'); + } + } + + // Create the channel + try { + Meteor.runAsUser(creatorId, () => { + const roomInfo = roomData.t === 'd' + ? Meteor.call('createDirectMessage', ...members) + : Meteor.call(roomData.t === 'p' ? 'createPrivateGroup' : 'createChannel', roomData.name, members); + + roomData._id = roomInfo.rid; + }); + } catch (e) { + console.warn(roomData.name, members); + console.error(e); + throw e; + } + + this.updateRoomId(roomData._id as 'string', roomData); + } + + convertImportedIdsToUsernames(importedIds: Array, idToRemove: string | undefined = undefined): Array { + return importedIds.map((user) => { + if (user === 'rocket.cat') { + return user; + } + + if (this._userCache.has(user)) { + const cache = this._userCache.get(user); + if (cache) { + return cache.username; + } + } + + const obj = Users.findOneByImportId(user, { fields: { _id: 1, username: 1 } }); + if (obj) { + this.addUserToCache(user, obj._id, obj.username); + + if (idToRemove && obj._id === idToRemove) { + return false; + } + + return obj.username; + } + + return false; + }).filter((user) => user); + } + + findExistingRoom(data: IImportChannel): IRoom { + if (data._id && data._id.toUpperCase() === 'GENERAL') { + const room = Rooms.findOneById('GENERAL', {}); + // Prevent the importer from trying to create a new general + if (!room) { + throw new Error('importer-channel-general-not-found'); + } + + return room; + } + + if (data.t === 'd') { + const users = this.convertImportedIdsToUsernames(data.users); + if (users.length !== data.users.length) { + throw new Error('importer-channel-missing-users'); + } + + return Rooms.findDirectRoomContainingAllUsernames(users, {}); + } + + return Rooms.findOneByNonValidatedName(data.name, {}); + } + + convertChannels(startedByUserId: string, { beforeImportFn, afterImportFn }: IConversionCallbacks = {}): void { + const channels = ImportData.find({ dataType: 'channel' }); + channels.forEach(({ data: c, _id }: IImportChannelRecord) => { + try { + if (beforeImportFn && !beforeImportFn(c, 'channel')) { + this.skipRecord(_id); + return; + } + + if (!c.name && c.t !== 'd') { + throw new Error('importer-channel-missing-name'); + } + + c.importIds = c.importIds.filter((item) => item); + c.users = _.uniq(c.users); + + if (!c.importIds.length) { + throw new Error('importer-channel-missing-import-id'); + } + + const existingRoom = this.findExistingRoom(c); + + if (existingRoom) { + this.updateRoom(existingRoom, c, startedByUserId); + } else { + this.insertRoom(c, startedByUserId); + } + + if (c.archived && c._id) { + this.archiveRoomById(c._id); + } + + if (afterImportFn) { + afterImportFn(c, 'channel'); + } + } catch (e) { + this.saveError(_id, e); + } + }); + } + + archiveRoomById(rid: string): void { + Rooms.archiveById(rid); + Subscriptions.archiveByRoomId(rid); + } + + convertData(startedByUserId: string, callbacks: IConversionCallbacks = {}): void { + this.convertUsers(callbacks); + this.convertChannels(startedByUserId, callbacks); + this.convertMessages(callbacks); + + Meteor.defer(() => { + this.clearSuccessfullyImportedData(); + }); + } + + clearImportData(): void { + const rawCollection = ImportData.model.rawCollection(); + const remove = Meteor.wrapAsync(rawCollection.remove, rawCollection); + + remove({}); + } + + clearSuccessfullyImportedData(): void { + ImportData.model.rawCollection().remove({ + errors: { + $exists: false, + }, + }); + } +} diff --git a/app/importer/server/classes/ImporterBase.js b/app/importer/server/classes/ImporterBase.js index df2052eb8d6d3..2d868b5b6e59b 100644 --- a/app/importer/server/classes/ImporterBase.js +++ b/app/importer/server/classes/ImporterBase.js @@ -7,67 +7,25 @@ import AdmZip from 'adm-zip'; import getFileType from 'file-type'; import { Progress } from './ImporterProgress'; -import { Selection } from './ImporterSelection'; import { ImporterWebsocket } from './ImporterWebsocket'; import { ProgressStep } from '../../lib/ImporterProgressStep'; import { ImporterInfo } from '../../lib/ImporterInfo'; import { RawImports } from '../models/RawImports'; import { Settings, Imports } from '../../../models'; import { Logger } from '../../../logger'; -import { FileUpload } from '../../../file-upload'; -import { sendMessage } from '../../../lib'; +import { ImportDataConverter } from './ImportDataConverter'; +import { ImportData } from '../models/ImportData'; +import { t } from '../../../utils/server'; +import { + Selection, + SelectionChannel, + SelectionUser, +} from '..'; /** * Base class for all of the importers. */ export class Base { - /** - * The max BSON object size we can store in MongoDB is 16777216 bytes - * but for some reason the mongo instanace which comes with Meteor - * errors out for anything close to that size. So, we are rounding it - * down to 8000000 bytes. - * - * @param {any} item The item to calculate the BSON size of. - * @returns {number} The size of the item passed in. - * @static - */ - static getBSONSize(item) { - const { calculateObjectSize } = require('bson'); - - return calculateObjectSize(item); - } - - /** - * The max BSON object size we can store in MongoDB is 16777216 bytes - * but for some reason the mongo instanace which comes with Meteor - * errors out for anything close to that size. So, we are rounding it - * down to 6000000 bytes. - * - * @returns {number} 8000000 bytes. - */ - static getMaxBSONSize() { - return 6000000; - } - - /** - * Splits the passed in array to at least one array which has a size that - * is safe to store in the database. - * - * @param {any[]} theArray The array to split out - * @returns {any[][]} The safe sized arrays - * @static - */ - static getBSONSafeArraysFromAnArray(theArray) { - const BSONSize = Base.getBSONSize(theArray); - const maxSize = Math.floor(theArray.length / Math.ceil(BSONSize / Base.getMaxBSONSize())); - const safeArrays = []; - let i = 0; - while (i < theArray.length) { - safeArrays.push(theArray.slice(i, i += maxSize)); - } - return safeArrays; - } - /** * Constructs a new importer, adding an empty collection, AdmZip property, and empty users & channels * @@ -84,6 +42,7 @@ export class Base { this.https = https; this.AdmZip = AdmZip; this.getFileType = getFileType; + this.converter = new ImportDataConverter(); this.prepare = this.prepare.bind(this); this.startImport = this.startImport.bind(this); @@ -92,7 +51,6 @@ export class Base { this.addCountToTotal = this.addCountToTotal.bind(this); this.addCountCompleted = this.addCountCompleted.bind(this); this.updateRecord = this.updateRecord.bind(this); - this.uploadFile = this.uploadFile.bind(this); this.info = info; @@ -196,7 +154,71 @@ export class Base { throw new Error(`Channels in the selected data wasn't found, it must but at least an empty array for the ${ this.info.name } importer.`); } - return this.updateProgress(ProgressStep.IMPORTING_STARTED); + this.updateProgress(ProgressStep.IMPORTING_STARTED); + this.reloadCount(); + const started = Date.now(); + const startedByUserId = Meteor.userId(); + console.log(importSelection); + + const beforeImportFn = (data, type) => { + switch (type) { + case 'channel': { + const id = data.t === 'd' ? '__directMessages__' : data.importIds[0]; + for (const channel of importSelection.channels) { + if (channel.channel_id === id) { + return channel.do_import; + } + } + + return false; + } + case 'user': { + const id = data.importIds[0]; + for (const user of importSelection.users) { + if (user.user_id === id) { + return user.do_import; + } + } + + return false; + } + } + + return true; + }; + + const afterImportFn = () => { + this.addCountCompleted(1); + }; + + Meteor.defer(() => { + try { + this.updateProgress(ProgressStep.IMPORTING_USERS); + this.converter.convertUsers({ beforeImportFn, afterImportFn }); + + this.updateProgress(ProgressStep.IMPORTING_CHANNELS); + this.converter.convertChannels(startedByUserId, { beforeImportFn, afterImportFn }); + + this.updateProgress(ProgressStep.IMPORTING_MESSAGES); + this.converter.convertMessages({ afterImportFn }); + + this.updateProgress(ProgressStep.FINISHING); + + Meteor.defer(() => { + this.converter.clearSuccessfullyImportedData(); + }); + + this.updateProgress(ProgressStep.DONE); + } catch (e) { + this.logger.error(e); + this.updateProgress(ProgressStep.ERROR); + } + + const timeTook = Date.now() - started; + this.logger.log(`Import took ${ timeTook } milliseconds.`); + }); + + return this.getProgress(); } /** @@ -352,18 +374,6 @@ export class Base { }); } - flagConflictingEmails(emailList) { - Imports.model.update({ - _id: this.importRecord._id, - 'fileData.users.email': { $in: emailList }, - }, { - $set: { - 'fileData.users.$.is_email_taken': true, - 'fileData.users.$.do_import': false, - }, - }); - } - /** * Updates the import record with the given fields being `set`. * @@ -377,79 +387,23 @@ export class Base { return this.importRecord; } - /** - * Uploads the file to the storage. - * - * @param {any} details An object with details about the upload: `name`, `size`, `type`, and `rid`. - * @param {string} fileUrl Url of the file to download/import. - * @param {any} user The Rocket.Chat user. - * @param {any} room The Rocket.Chat Room. - * @param {Date} timeStamp The timestamp the file was uploaded - */ - uploadFile(details, fileUrl, user, room, timeStamp) { - this.logger.debug(`Uploading the file ${ details.name } from ${ fileUrl }.`); - const requestModule = /https/i.test(fileUrl) ? this.https : this.http; - - const fileStore = FileUpload.getStore('Uploads'); - - return requestModule.get(fileUrl, Meteor.bindEnvironment(function(res) { - const contentType = res.headers['content-type']; - if (!details.type && contentType) { - details.type = contentType; - } + buildSelection() { + this.updateProgress(ProgressStep.USER_SELECTION); - const rawData = []; - res.on('data', (chunk) => rawData.push(chunk)); - res.on('end', Meteor.bindEnvironment(() => { - fileStore.insert(details, Buffer.concat(rawData), function(err, file) { - if (err) { - throw new Error(err); - } else { - const url = FileUpload.getPath(`${ file._id }/${ encodeURI(file.name) }`); - - const attachment = { - title: file.name, - title_link: url, - }; - - if (/^image\/.+/.test(file.type)) { - attachment.image_url = url; - attachment.image_type = file.type; - attachment.image_size = file.size; - attachment.image_dimensions = file.identify != null ? file.identify.size : undefined; - } + const users = ImportData.getAllUsersForSelection(); + const channels = ImportData.getAllChannelsForSelection(); + const hasDM = ImportData.checkIfDirectMessagesExists(); - if (/^audio\/.+/.test(file.type)) { - attachment.audio_url = url; - attachment.audio_type = file.type; - attachment.audio_size = file.size; - } + const selectionUsers = users.map((u) => new SelectionUser(u.data.importIds[0], u.data.username, u.data.emails[0], Boolean(u.data.deleted), u.data.type === 'bot', true)); + const selectionChannels = channels.map((c) => new SelectionChannel(c.data.importIds[0], c.data.name, Boolean(c.data.archived), true, c.data.t === 'p', undefined, c.data.t === 'd')); + const selectionMessages = ImportData.countMessages(); - if (/^video\/.+/.test(file.type)) { - attachment.video_url = url; - attachment.video_type = file.type; - attachment.video_size = file.size; - } + if (hasDM) { + selectionChannels.push(new SelectionChannel('__directMessages__', t('Direct_Messages'), false, true, true, undefined, true)); + } - const msg = { - rid: details.rid, - ts: timeStamp, - msg: '', - file: { - _id: file._id, - }, - groupable: false, - attachments: [attachment], - }; - - if ((details.message_id != null) && (typeof details.message_id === 'string')) { - msg._id = details.message_id; - } + const results = new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); - return sendMessage(user, msg, room, true); - } - }); - })); - })); + return results; } } diff --git a/app/importer/server/definitions/IImportChannel.ts b/app/importer/server/definitions/IImportChannel.ts new file mode 100644 index 0000000000000..5ad24ed99afbb --- /dev/null +++ b/app/importer/server/definitions/IImportChannel.ts @@ -0,0 +1,14 @@ +export interface IImportChannel { + _id?: string; + u?: { + _id: string; + }; + name?: string; + users: Array; + importIds: Array; + t: string; + topic?: string; + description?: string; + ts?: Date; + archived?: boolean; +} diff --git a/app/importer/server/definitions/IImportMessage.ts b/app/importer/server/definitions/IImportMessage.ts new file mode 100644 index 0000000000000..ae5357bec4822 --- /dev/null +++ b/app/importer/server/definitions/IImportMessage.ts @@ -0,0 +1,53 @@ +export type IImportedId = 'string'; + +export interface IImportMessageReaction { + name: string; + users: Array; +} + +export interface IImportPendingFile { + downloadUrl: string; + id: string; + size: number; + name: string; + external: boolean; + source: string; + original: Record; +} + +export interface IImportAttachment extends Record { + text: string; + title: string; + fallback: string; +} + +export interface IImportMessage { + _id?: IImportedId; + + rid: IImportedId; + u: { + _id: IImportedId; + }; + + msg: string; + alias?: string; + ts: Date; + t?: string; + reactions?: Record; + groupable?: boolean; + + tmid?: IImportedId; + tlm?: Date; + tcount?: number; + replies?: Array; + editedAt?: Date; + editedBy?: IImportedId; + mentions?: Array; + channels?: Array; + attachments?: IImportAttachment; + bot?: boolean; + emoji?: string; + + url?: string; + _importFile?: IImportPendingFile; +} diff --git a/app/importer/server/definitions/IImportRecord.ts b/app/importer/server/definitions/IImportRecord.ts new file mode 100644 index 0000000000000..09d3a9418ef46 --- /dev/null +++ b/app/importer/server/definitions/IImportRecord.ts @@ -0,0 +1,28 @@ +import { IImportUser } from './IImportUser'; +import { IImportChannel } from './IImportChannel'; +import { IImportMessage } from './IImportMessage'; + +export interface IImportRecord { + data: IImportUser | IImportChannel | IImportMessage; + dataType: 'user' | 'channel' | 'message'; + _id: string; + options?: {}; +} + +export interface IImportUserRecord extends IImportRecord { + data: IImportUser; + dataType: 'user'; +} + +export interface IImportChannelRecord extends IImportRecord { + data: IImportChannel; + dataType: 'channel'; +} + +export interface IImportMessageRecord extends IImportRecord { + data: IImportMessage; + dataType: 'message'; + options: { + useQuickInsert?: boolean; + }; +} diff --git a/app/importer/server/definitions/IImportUser.ts b/app/importer/server/definitions/IImportUser.ts new file mode 100644 index 0000000000000..6462cb054a9ea --- /dev/null +++ b/app/importer/server/definitions/IImportUser.ts @@ -0,0 +1,17 @@ +export interface IImportUser { + // #ToDo: Remove this _id, as it isn't part of the imported data + _id?: string; + + username?: string; + emails: Array; + importIds: Array; + name?: string; + utcOffset?: number; + active?: boolean; + avatarUrl?: string; + deleted?: boolean; + statusText?: string; + roles?: Array; + type: 'user' | 'bot'; + bio?: string; +} diff --git a/app/importer/server/index.js b/app/importer/server/index.js index 0fed91e5b6608..dd9e89ba02096 100644 --- a/app/importer/server/index.js +++ b/app/importer/server/index.js @@ -2,6 +2,7 @@ import { Base } from './classes/ImporterBase'; import { ImporterWebsocket } from './classes/ImporterWebsocket'; import { Progress } from './classes/ImporterProgress'; import { RawImports } from './models/RawImports'; +import { ImportData } from './models/ImportData'; import { Selection } from './classes/ImporterSelection'; import { SelectionChannel } from './classes/ImporterSelectionChannel'; import { SelectionUser } from './classes/ImporterSelectionUser'; @@ -25,6 +26,7 @@ export { Progress, ProgressStep, RawImports, + ImportData, Selection, SelectionChannel, SelectionUser, diff --git a/app/importer/server/methods/getImportFileData.js b/app/importer/server/methods/getImportFileData.js index f1029b9869f18..e49f2a903dd76 100644 --- a/app/importer/server/methods/getImportFileData.js +++ b/app/importer/server/methods/getImportFileData.js @@ -58,31 +58,17 @@ Meteor.methods({ ]; if (readySteps.indexOf(importer.instance.progress.step) >= 0) { - if (importer.instance.importRecord && importer.instance.importRecord.fileData) { - return importer.instance.importRecord.fileData; - } + return importer.instance.buildSelection(); } const fileName = importer.instance.importRecord.file; const fullFilePath = fs.existsSync(fileName) ? fileName : path.join(RocketChatImportFileInstance.absolutePath, fileName); - const results = importer.instance.prepareUsingLocalFile(fullFilePath); - - if (results instanceof Promise) { - return results.then((data) => { - importer.instance.updateRecord({ - fileData: data, - }); - - return data; - }).catch((e) => { - console.error(e); - throw new Meteor.Error(e); - }); + const promise = importer.instance.prepareUsingLocalFile(fullFilePath); + + if (promise && promise instanceof Promise) { + Promise.await(promise); } - importer.instance.updateRecord({ - fileData: results, - }); - return results; + return importer.instance.buildSelection(); }, }); diff --git a/app/importer/server/models/ImportData.ts b/app/importer/server/models/ImportData.ts new file mode 100644 index 0000000000000..a6afb291e19c1 --- /dev/null +++ b/app/importer/server/models/ImportData.ts @@ -0,0 +1,88 @@ +import { Base } from '../../../models/server'; +import { IImportUserRecord, IImportChannelRecord } from '../definitions/IImportRecord'; + +class ImportDataModel extends Base { + constructor() { + super('import_data'); + } + + getAllUsersForSelection(): Array { + return this.find({ + dataType: 'user', + }, { + fields: { + 'data.importIds': 1, + 'data.username': 1, + 'data.emails': 1, + 'data.deleted': 1, + 'data.type': 1, + }, + }).fetch(); + } + + getAllChannelsForSelection(): Array { + return this.find({ + dataType: 'channel', + 'data.t': { + $ne: 'd', + }, + }, { + fields: { + 'data.importIds': 1, + 'data.name': 1, + 'data.archived': 1, + 'data.t': 1, + }, + }).fetch(); + } + + checkIfDirectMessagesExists(): boolean { + return this.find({ + dataType: 'channel', + 'data.t': 'd', + }, { + fields: { + _id: 1, + }, + }).count() > 0; + } + + countMessages(): number { + return this.find({ + dataType: 'message', + }).count(); + } + + findChannelImportIdByNameOrImportId(channelIdentifier: string): string | undefined { + const channel = this.findOne({ + dataType: 'channel', + $or: [ + { + 'data.name': channelIdentifier, + }, + { + 'data.importIds': channelIdentifier, + }, + ], + }, { + fields: { + 'data.importIds': 1, + }, + }); + + return channel?.data?.importIds?.shift(); + } + + findDMForImportedUsers(...users: Array): IImportChannelRecord | undefined { + const query = { + dataType: 'channel', + 'data.users': { + $all: users, + }, + }; + + return this.findOne(query); + } +} + +export const ImportData = new ImportDataModel(); diff --git a/app/importer/server/startup/setImportsToInvalid.js b/app/importer/server/startup/setImportsToInvalid.js index c5303516cb597..a7ec863732225 100644 --- a/app/importer/server/startup/setImportsToInvalid.js +++ b/app/importer/server/startup/setImportsToInvalid.js @@ -8,7 +8,7 @@ function runDrop(fn) { try { fn(); } catch (e) { - console.log('errror', e); // TODO: Remove + console.log('error', e); // TODO: Remove // ignored } } @@ -21,9 +21,7 @@ Meteor.startup(function() { // And there's still data for it on the temp collection // Then we can keep the data there to let the user try again if (lastOperation && [ProgressStep.USER_SELECTION, ProgressStep.ERROR].includes(lastOperation.status)) { - if (RawImports.find({ import: lastOperation._id }).count() > 0) { - idToKeep = lastOperation._id; - } + idToKeep = lastOperation._id; } if (idToKeep) { diff --git a/app/lib/server/functions/insertMessage.js b/app/lib/server/functions/insertMessage.js index d45b8dcc275d7..5e342ad0ab7c6 100644 --- a/app/lib/server/functions/insertMessage.js +++ b/app/lib/server/functions/insertMessage.js @@ -76,8 +76,8 @@ const validateAttachment = (attachment) => { const validateBodyAttachments = (attachments) => attachments.map(validateAttachment); -export const insertMessage = function(user, message, room, upsert = false) { - if (!user || !message || !room._id) { +export const insertMessage = function(user, message, rid, upsert = false) { + if (!user || !message || !rid) { return false; } @@ -103,7 +103,7 @@ export const insertMessage = function(user, message, room, upsert = false) { _id, username, }; - message.rid = room._id; + message.rid = rid; if (!Match.test(message.msg, String)) { message.msg = ''; diff --git a/app/models/server/models/Rooms.js b/app/models/server/models/Rooms.js index 3208a1130add5..4b7ed177e01dd 100644 --- a/app/models/server/models/Rooms.js +++ b/app/models/server/models/Rooms.js @@ -926,7 +926,7 @@ export class Rooms extends Base { return this.update(query, update); } - resetLastMessageById(_id, messageId) { + resetLastMessageById(_id, messageId = undefined) { const query = { _id }; const lastMessage = Messages.getLastVisibleMessageSentWithNoTypeByRoomId(_id, messageId); diff --git a/client/importPackages.ts b/client/importPackages.ts index 06f1a470051c8..e71d31b426032 100644 --- a/client/importPackages.ts +++ b/client/importPackages.ts @@ -21,7 +21,6 @@ import '../app/google-vision/client'; import '../app/iframe-login/client'; import '../app/importer/client'; import '../app/importer-csv/client'; -import '../app/importer-hipchat/client'; import '../app/importer-hipchat-enterprise/client'; import '../app/importer-slack/client'; import '../app/importer-slack-users/client'; diff --git a/client/views/admin/import/ImportHistoryPage.js b/client/views/admin/import/ImportHistoryPage.js index 28b271ba4db63..3487ae12fc76e 100644 --- a/client/views/admin/import/ImportHistoryPage.js +++ b/client/views/admin/import/ImportHistoryPage.js @@ -58,11 +58,8 @@ function ImportHistoryPage() { t, ]); - const hasAnySuccessfulSlackImport = useMemo( - () => - latestOperations?.some( - ({ importerKey, status }) => importerKey === 'slack' && status === ProgressStep.DONE, - ), + const hasAnySuccessfulImport = useMemo( + () => latestOperations?.some(({ status }) => status === ProgressStep.DONE), [latestOperations], ); @@ -119,12 +116,12 @@ function ImportHistoryPage() { - {hasAnySuccessfulSlackImport && ( + {hasAnySuccessfulImport && ( )} - {hasAnySuccessfulSlackImport && ( + {hasAnySuccessfulImport && ( diff --git a/server/importPackages.js b/server/importPackages.js index 412bb39fd3877..5e28b6d51ba68 100644 --- a/server/importPackages.js +++ b/server/importPackages.js @@ -31,7 +31,6 @@ import '../app/google-vision/server'; import '../app/iframe-login/server'; import '../app/importer/server'; import '../app/importer-csv/server'; -import '../app/importer-hipchat/server'; import '../app/importer-hipchat-enterprise/server'; import '../app/importer-pending-files/server'; import '../app/importer-pending-avatars/server';