Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ability to run imports several times without duplicate messages #3123

Merged
merged 5 commits into from
May 6, 2016
Merged

Ability to run imports several times without duplicate messages #3123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
716eb74
Continue work started by patcon in #2272 for importing slack again an…
graywolf336 May 3, 2016
b99df14
Merge branch 'develop' into incremental-slack-import
graywolf336 May 6, 2016
4c348c2
Ignore new slack channel format
rodrigok May 6, 2016
a2e96f8
Add ability to upsert messages on sendMessage
rodrigok May 6, 2016
db7283a
Show a better error for duplicated oembed records
rodrigok May 6, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/rocketchat-importer-hipchat/package.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Package.onUse(function(api) {
'rocketchat:[email protected]'
]);
api.use(['mrt:[email protected]'], 'server');

api.use('rocketchat:logger', 'server');
api.addFiles('server.coffee', 'server');
api.addFiles('main.coffee', ['client', 'server']);
});
41 changes: 18 additions & 23 deletions packages/rocketchat-importer-hipchat/server.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base

constructor: (name, descriptionI18N, fileTypeRegex) ->
super(name, descriptionI18N, fileTypeRegex)
@logger.debug('Constructed a new Slack Importer.')
@userTags = []

prepare: (dataURI, sentContentType, fileName) =>
super(dataURI, sentContentType, fileName)

# try
{image, contentType} = RocketChatFile.dataURIParse dataURI
zip = new @AdmZip(new Buffer(image, 'base64'))
zipEntries = zip.getEntries()
Expand All @@ -21,7 +21,7 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base
do (entry) =>
if entry.entryName.indexOf('__MACOSX') > -1
#ignore all of the files inside of __MACOSX
console.log("Ignoring the file: #{entry.entryName}")
@logger.debug("Ignoring the file: #{entry.entryName}")
if not entry.isDirectory
if entry.entryName.indexOf(Importer.HipChat.RoomPrefix) > -1
roomName = entry.entryName.split(Importer.HipChat.RoomPrefix)[1]
Expand All @@ -41,14 +41,14 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base
try
tempMessages[roomName][msgGroupData] = JSON.parse entry.getData().toString()
catch
console.warn "#{entry.entryName} is not a valid JSON file! Unable to import it."
@logger.warn "#{entry.entryName} is not a valid JSON file! Unable to import it."
else if entry.entryName.indexOf(Importer.HipChat.UsersPrefix) > -1
usersName = entry.entryName.split(Importer.HipChat.UsersPrefix)[1]
if usersName is 'list.json'
@updateProgress Importer.ProgressStep.PREPARING_USERS
tempUsers = JSON.parse(entry.getData().toString()).users
else
console.warn "Unexpected file in the #{@name} import: #{entry.entryName}"
@logger.warn "Unexpected file in the #{@name} import: #{entry.entryName}"

# Insert the users record, eventually this might have to be split into several ones as well
# if someone tries to import a several thousands users instance
Expand Down Expand Up @@ -86,6 +86,7 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base
@addCountToTotal messagesCount

if tempUsers.length is 0 or tempRooms.length is 0 or messagesCount is 0
@logger.warn "The loaded users count #{tempUsers.length}, the loaded channels #{tempChannels.length}, and the loaded messages #{messagesCount}"
@updateProgress Importer.ProgressStep.ERROR
return @getProgress()

Expand All @@ -97,10 +98,6 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base

@updateProgress Importer.ProgressStep.USER_SELECTION
return new Importer.Selection @name, selectionUsers, selectionChannels
# catch error
# @updateRecord { 'failed': true, 'error': error }
# console.error Importer.ProgressStep.ERROR
# throw new Error 'import-hipchat-error', error

startImport: (importSelection) =>
super(importSelection)
Expand All @@ -118,7 +115,6 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base

startedByUserId = Meteor.userId()
Meteor.defer =>
# try
@updateProgress Importer.ProgressStep.IMPORTING_USERS
for user in @users.users when user.do_import
do (user) =>
Expand Down Expand Up @@ -163,13 +159,14 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base
for user in @users.users when user.user_id is channel.owner_user_id
userId = user.rocketId

if userId isnt ''
Meteor.runAsUser userId, () =>
returned = Meteor.call 'createChannel', channel.name, []
channel.rocketId = returned.rid
RocketChat.models.Rooms.update { _id: channel.rocketId }, { $set: { 'ts': new Date(channel.created * 1000) }}
else
console.warn "Failed to find the channel creator for #{channel.name}."
if userId is ''
@logger.warn "Failed to find the channel creator for #{channel.name}, setting it to the current running user."
userId = startedByUserId

Meteor.runAsUser userId, () =>
returned = Meteor.call 'createChannel', channel.name, []
channel.rocketId = returned.rid
RocketChat.models.Rooms.update { _id: channel.rocketId }, { $set: { 'ts': new Date(channel.created * 1000) }}
@addCountCompleted 1
@collection.update { _id: @channels._id }, { $set: { 'channels': @channels.channels }}

Expand Down Expand Up @@ -202,7 +199,7 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base
if not _.isArray message
console.warn 'Please report the following:', message
@addCountCompleted 1
console.warn 'The following did not have users:', nousers
@logger.warn 'The following did not have users:', nousers

@updateProgress Importer.ProgressStep.FINISHING
for channel in @channels.channels when channel.do_import and channel.is_archived
Expand All @@ -212,11 +209,7 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base

@updateProgress Importer.ProgressStep.DONE
timeTook = Date.now() - start
console.log "Import took #{timeTook} milliseconds."
# catch error
# @updateRecord { 'failed': true, 'error': error }
# console.error Importer.ProgressStep.ERROR
# throw new Error 'import-hipchat-error', error
@logger.log "Import took #{timeTook} milliseconds."

return @getProgress()

Expand All @@ -232,7 +225,9 @@ Importer.HipChat = class Importer.HipChat extends Importer.Base
if message?
for userReplace in @userTags
message = message.replace userReplace.hipchat, userReplace.rocket
return message
else
message = ''
return message

getSelection: () =>
selectionUsers = @users.users.map (user) ->
Expand Down
1 change: 1 addition & 0 deletions packages/rocketchat-importer-slack/package.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ Package.onUse(function(api) {
'rocketchat:[email protected]'
]);
api.use(['mrt:[email protected]'], 'server');
api.use('rocketchat:logger', 'server');
api.addFiles('server.coffee', 'server');
api.addFiles('main.coffee', ['client', 'server']);
});
71 changes: 39 additions & 32 deletions packages/rocketchat-importer-slack/server.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ Importer.Slack = class Importer.Slack extends Importer.Base
super(name, descriptionI18N, fileTypeRegex)
@userTags = []
@bots = {}
@logger.debug('Constructed a new Slack Importer.')

prepare: (dataURI, sentContentType, fileName) =>
super(dataURI, sentContentType, fileName)

# try
{image, contentType} = RocketChatFile.dataURIParse dataURI
zip = new @AdmZip(new Buffer(image, 'base64'))
zipEntries = zip.getEntries()
Expand All @@ -19,10 +19,11 @@ Importer.Slack = class Importer.Slack extends Importer.Base
do (entry) =>
if entry.entryName.indexOf('__MACOSX') > -1
#ignore all of the files inside of __MACOSX
console.log("Ignoring the file: #{entry.entryName}")
@logger.debug("Ignoring the file: #{entry.entryName}")
else if entry.entryName == 'channels.json'
@updateProgress Importer.ProgressStep.PREPARING_CHANNELS
tempChannels = JSON.parse entry.getData().toString()
tempChannels = tempChannels.filter (channel) -> channel.creator?
else if entry.entryName == 'users.json'
@updateProgress Importer.ProgressStep.PREPARING_USERS
tempUsers = JSON.parse entry.getData().toString()
Expand All @@ -40,7 +41,7 @@ Importer.Slack = class Importer.Slack extends Importer.Base
try
tempMessages[channelName][msgGroupData] = JSON.parse entry.getData().toString()
catch
console.warn "#{entry.entryName} is not a valid JSON file! Unable to import it."
@logger.warn "#{entry.entryName} is not a valid JSON file! Unable to import it."

# Insert the users record, eventually this might have to be split into several ones as well
# if someone tries to import a several thousands users instance
Expand Down Expand Up @@ -78,7 +79,7 @@ Importer.Slack = class Importer.Slack extends Importer.Base
@addCountToTotal messagesCount

if tempUsers.length is 0 or tempChannels.length is 0 or messagesCount is 0
console.log "The loaded users count #{tempUsers.length}, the loaded channels #{tempChannels.length}, and the loaded messages #{messagesCount}"
@logger.warn "The loaded users count #{tempUsers.length}, the loaded channels #{tempChannels.length}, and the loaded messages #{messagesCount}"
@updateProgress Importer.ProgressStep.ERROR
return @getProgress()

Expand All @@ -89,10 +90,6 @@ Importer.Slack = class Importer.Slack extends Importer.Base

@updateProgress Importer.ProgressStep.USER_SELECTION
return new Importer.Selection @name, selectionUsers, selectionChannels
# catch error
# @updateRecord { 'failed': true, 'error': error }
# console.error Importer.ProgressStep.ERROR
# throw new Error 'import-slack-error', error

startImport: (importSelection) =>
super(importSelection)
Expand All @@ -110,7 +107,6 @@ Importer.Slack = class Importer.Slack extends Importer.Base

startedByUserId = Meteor.userId()
Meteor.defer =>
# try
@updateProgress Importer.ProgressStep.IMPORTING_USERS
for user in @users.users when user.do_import
do (user) =>
Expand Down Expand Up @@ -171,6 +167,10 @@ Importer.Slack = class Importer.Slack extends Importer.Base
for user in @users.users when user.id is channel.creator
userId = user.rocketId

if userId is ''
@logger.warn "Failed to find the channel creator for #{channel.name}, setting it to the current running user."
userId = startedByUserId

Meteor.runAsUser userId, () =>
returned = Meteor.call 'createChannel', channel.name, users
channel.rocketId = returned.rid
Expand Down Expand Up @@ -203,55 +203,64 @@ Importer.Slack = class Importer.Slack extends Importer.Base
for date, msgs of messagesObj
@updateRecord { 'messagesstatus': "#{channel}/#{date}.#{msgs.messages.length}" }
for message in msgs.messages
msgDataDefaults =
_id: "S#{message.ts}"
ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)

if message.type is 'message'
if message.subtype?
if message.subtype is 'channel_join'
if @getRocketUser(message.user)?
RocketChat.models.Messages.createUserJoinWithRoomIdAndUser room._id, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
RocketChat.models.Messages.createUserJoinWithRoomIdAndUser room._id, @getRocketUser(message.user), msgDataDefaults
else if message.subtype is 'channel_leave'
if @getRocketUser(message.user)?
RocketChat.models.Messages.createUserLeaveWithRoomIdAndUser room._id, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
RocketChat.models.Messages.createUserLeaveWithRoomIdAndUser room._id, @getRocketUser(message.user), msgDataDefaults
else if message.subtype is 'me_message'
RocketChat.sendMessage @getRocketUser(message.user), { msg: '_' + @convertSlackMessageToRocketChat(message.text) + '_', ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }, room
msgObj =
msg: "_#{@convertSlackMessageToRocketChat(message.text)}_"
_.extend msgObj, msgDataDefaults
RocketChat.sendMessage @getRocketUser(message.user), msgObj, room
else if message.subtype is 'bot_message'
botUser = RocketChat.models.Users.findOneById 'rocket.cat', { fields: { username: 1 }}
botUsername = if @bots[message.bot_id] then @bots[message.bot_id]?.name else message.username
msgObj =
msg: if message.text then message.text else ''
ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
msg: @convertSlackMessageToRocketChat(message.text)
rid: room._id
bot: true
attachments: message.attachments
username: if botUsername then botUsername else undefined

_.extend msgObj, msgDataDefaults

if message.edited?
msgObj.ets = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000)

if message.icons?
msgObj.emoji = message.icons.emoji

msgObj.msg = @convertSlackMessageToRocketChat(msgObj.msg)

RocketChat.sendMessage botUser, msgObj, room
RocketChat.sendMessage botUser, msgObj, room, upsert: true
else if message.subtype is 'channel_purpose'
RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser 'room_changed_topic', room._id, message.purpose, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser 'room_changed_topic', room._id, message.purpose, @getRocketUser(message.user), msgDataDefaults
else if message.subtype is 'channel_topic'
RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser 'room_changed_topic', room._id, message.topic, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser 'room_changed_topic', room._id, message.topic, @getRocketUser(message.user), msgDataDefaults
else if message.subtype is 'pinned_item'
if message.attachments
RocketChat.models.Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUser(message.user),
ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
msgObj =
attachments: [
"text" : @convertSlackMessageToRocketChat message.attachments[0].text
"author_name" : message.attachments[0].author_subname
"author_icon" : getAvatarUrlFromUsername(message.attachments[0].author_subname)
]
_.extend msgObj, msgDataDefaults
RocketChat.models.Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUser(message.user), msgObj
else
RocketChat.models.Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUser(message.user),
ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
#TODO: make this better
@logger.debug('Pinned item with no attachment, needs work.');
#RocketChat.models.Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUser(message.user), msgDataDefaults
else if message.subtype is 'file_share'
if message.file?.url_private_download isnt undefined
details =
message_id: "S#{message.ts}"
name: message.file.name
size: message.file.size
type: message.file.mimetype
Expand All @@ -265,16 +274,17 @@ Importer.Slack = class Importer.Slack extends Importer.Base
if user?
msgObj =
msg: @convertSlackMessageToRocketChat message.text
ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
rid: room._id
u:
_id: user._id
username: user.username

_.extend msgObj, msgDataDefaults

if message.edited?
msgObj.ets = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000)

RocketChat.sendMessage @getRocketUser(message.user), msgObj, room
RocketChat.sendMessage @getRocketUser(message.user), msgObj, room, upsert: true
@addCountCompleted 1
console.log missedTypes
@updateProgress Importer.ProgressStep.FINISHING
Expand All @@ -285,12 +295,7 @@ Importer.Slack = class Importer.Slack extends Importer.Base

@updateProgress Importer.ProgressStep.DONE
timeTook = Date.now() - start
console.log "Import took #{timeTook} milliseconds."
# catch error
# @updateRecord { 'failed': true, 'error': error }
# @updateProgress Importer.ProgressStep.ERROR
# console.error Importer.ProgressStep.ERROR
# throw new Error 'import-slack-error', error
@logger.log "Import took #{timeTook} milliseconds."

return @getProgress()

Expand All @@ -317,7 +322,9 @@ Importer.Slack = class Importer.Slack extends Importer.Base
for userReplace in @userTags
message = message.replace userReplace.slack, userReplace.rocket
message = message.replace userReplace.slackLong, userReplace.rocket
return message
else
message = ''
return message

getSelection: () =>
selectionUsers = @users.users.map (user) ->
Expand Down
1 change: 1 addition & 0 deletions packages/rocketchat-importer/package.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Package.onUse(function(api) {
'rocketchat:[email protected]'
]);

api.use('rocketchat:logger', 'server');
api.use('templating', 'client');

//Import Framework
Expand Down
10 changes: 7 additions & 3 deletions packages/rocketchat-importer/server/classes/ImporterBase.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ Importer.Base = class Importer.Base
# @param [RegExp] fileTypeRegex the regexp to validate the uploaded file type against
#
constructor: (@name, @description, @fileTypeRegex) ->
@logger = new Logger("#{@name} Importer", {});
@progress = new Importer.Progress @name
@collection = Importer.RawImports
@AdmZip = Npm.require 'adm-zip'
Expand Down Expand Up @@ -100,7 +101,7 @@ Importer.Base = class Importer.Base
updateProgress: (step) =>
@progress.step = step

console.log "#{@name} is now at #{step}."
@logger.debug "#{@name} is now at #{step}."
@updateRecord { 'status': @progress.step }

return @progress
Expand Down Expand Up @@ -148,7 +149,7 @@ Importer.Base = class Importer.Base
# @param [Date] timeStamp the timestamp the file was uploaded
#
uploadFile: (details, fileUrl, user, room, timeStamp) =>
console.log "Uploading the file #{details.name} from #{fileUrl}."
@logger.debug "Uploading the file #{details.name} from #{fileUrl}."
requestModule = if /https/i.test(fileUrl) then Importer.Base.https else Importer.Base.http

requestModule.get fileUrl, Meteor.bindEnvironment((stream) ->
Expand Down Expand Up @@ -189,7 +190,10 @@ Importer.Base = class Importer.Base
groupable: false
attachments: [attachment]

if details.message_id? and (typeof details.message_id is 'string')
msg['_id'] = details.message_id

RocketChat.sendMessage user, msg, room
else
console.error "Failed to create the store for #{fileUrl}!!!"
@logger.error "Failed to create the store for #{fileUrl}!!!"
)
8 changes: 7 additions & 1 deletion packages/rocketchat-lib/server/functions/sendMessage.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,13 @@ RocketChat.sendMessage = (user, message, room) ->

message = RocketChat.callbacks.run 'beforeSaveMessage', message

message._id = RocketChat.models.Messages.insert message
if message._id?
_id = message._id
delete message._id
RocketChat.models.Messages.upsert {_id: _id, 'u._id': message.u._id}, message
message._id = _id
else
message._id = RocketChat.models.Messages.insert message

###
Defer other updates as their return is not interesting to the user
Expand Down
Loading