Improved livechat history migration to run faster

RocketChat · Jun 5, 2018 · b2535ea · b2535ea
1 parent bf90817
commit b2535ea
Showing 1 changed file with 78 additions and 39 deletions.
diff --git a/server/startup/migrations/v123.js b/server/startup/migrations/v123.js
@@ -1,48 +1,87 @@
+let pageVisitedCollection;
+let messageCollection;
+let roomCollection;
+
+const roomIdByToken = {};
+
+const batchSize = 5000;
+
+async function migrateHistory(total, current) {
+	console.log(`Livechat history migration ${ current }/${ total }`);
+
+	const items = await pageVisitedCollection.find({}).limit(batchSize).toArray();
+
+	const tokens = items.filter((item) => item.token && !roomIdByToken[item.token]).map((item) => item.token);
+	const rooms = await roomCollection.find({
+		'v.token': {
+			$in: tokens
+		}
+	}, {
+		fields: {
+			'v.token': 1
+		}
+	}).toArray();
+
+	rooms.forEach((room) => {
+		roomIdByToken[room.v.token] = room._id;
+	});
+
+	const actions = items.reduce((result, item) => {
+		const msg = {
+			t: 'livechat_navigation_history',
+			rid: roomIdByToken[item.token] || null, // prevent from being `undefined`
+			ts: item.ts,
+			msg: `${ item.page.title } - ${ item.page.location.href }`,
+			u: {
+				_id : 'rocket.cat',
+				username : 'rocket.cat'
+			},
+			groupable : false,
+			navigation : {
+				page: item.page,
+				token: item.token
+			}
+		};
+		if (!roomIdByToken[item.token] && item.expireAt) {
+			msg.expireAt = item.expireAt;
+		}
+		result.insert.push(msg);
+		result.remove.push(item._id);
+
+		return result;
+	}, { insert: [], remove: [] });
+
+	const batch = Promise.all([
+		messageCollection.insertMany(actions.insert),
+		pageVisitedCollection.removeMany({ _id: { $in: actions.remove } })
+	]);
+	if (actions.remove.length === batchSize) {
+		await batch;
+		return migrateHistory(total, current + batchSize);
+	}
+
+	return batch;
+}
+
+
 RocketChat.Migrations.add({
 	version: 123,
 	up() {
+		pageVisitedCollection = RocketChat.models.LivechatPageVisited.model.rawCollection();
+		messageCollection = RocketChat.models.Messages.model.rawCollection();
+		roomCollection = RocketChat.models.Rooms.model.rawCollection();
+
 		/*
 		 * Move visitor navigation history to messages
 		 */
-		let token = '';
-		let roomId;
-
-		console.log('Migrating livechat visitors navigation history to livechat messages ...');
-
-		Meteor.setTimeout(() => {
-			RocketChat.models.LivechatPageVisited.find({}).forEach((item) => {
-
-				if (token !== item.token) {
-					const rooms = RocketChat.models.Rooms.findByVisitorToken(item.token).fetch();
-					if ((rooms) && (rooms.length > 0)) {
-						roomId = rooms[0]._id;
-					} else {
-						roomId = null;
-					}
-					token = item.token;
-				}
-				if (roomId) {
-					const pageTitle = item.page.title;
-					const pageUrl = item.page.location.href;
-					const msg = {
-						t: 'livechat_navigation_history',
-						rid: roomId,
-						ts: item.ts,
-						msg: `${ pageTitle } - ${ pageUrl }`,
-						u: {
-							_id : 'rocket.cat',
-							username : 'rocket.cat'
-						},
-						groupable : false,
-						navigation : {
-							page: item.page,
-							token: item.token
-						}
-					};
-					RocketChat.models.Messages.insert(msg);
-				}
-				RocketChat.models.LivechatPageVisited.remove({_id: item._id});
-			});
+		Meteor.setTimeout(async() => {
+			const pages = pageVisitedCollection.find({});
+			const total = await pages.count();
+			await pages.close();
+
+			console.log('Migrating livechat visitors navigation history to livechat messages. This might take a long time ...');
+
+			await migrateHistory(total, 0);
 
 			console.log('Livechat visitors navigation history migration finished.');
 		}, 1000);