From aaf8b832449158e833beb7dfa13a38cda17aeb1c Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 12 Jan 2021 11:58:23 -0300 Subject: [PATCH 1/6] Add migration fix subs --- server/startup/migrations/index.js | 1 + server/startup/migrations/v213.js | 90 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 server/startup/migrations/v213.js diff --git a/server/startup/migrations/index.js b/server/startup/migrations/index.js index 9e53f75c3b96f..307fd414164c5 100644 --- a/server/startup/migrations/index.js +++ b/server/startup/migrations/index.js @@ -209,4 +209,5 @@ import './v209'; import './v210'; import './v211'; import './v212'; +import './v213'; import './xrun'; diff --git a/server/startup/migrations/v213.js b/server/startup/migrations/v213.js new file mode 100644 index 0000000000000..21cb3df07f026 --- /dev/null +++ b/server/startup/migrations/v213.js @@ -0,0 +1,90 @@ +import { Migrations } from '../../../app/migrations'; +import { Subscriptions, Rooms } from '../../../app/models/server/raw'; + +const batchSize = 2; + +const updateSubscriptions = async (total, current) => { + return new Promise(async (resolve, reject) => { + Subscriptions + .find({ t: 'd' }, { projection: { rid: 1, u: 1 } }) + .limit(batchSize) + .forEach(async (sub) => { + console.log('sub', sub); + const room = await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }); + if (!room) { + console.log(`ROOM NOT FOUND: ${ sub.rid }`); + return; + } + + if (!room.usernames || room.usernames.length === 0) { + console.log(`NO USERNAMES: ${ sub.rid }`); + return; + } + + const name = room.usernames + .filter((u) => u !== sub.u.username) + .sort() + .join(', ') || sub.u.username; + if (!name) { + console.log(`NO NAME ${ sub._id }`); + return; + } + + console.log('fix', sub._id); + await Subscriptions.update({ _id: sub._id }, { $set: { name, _updatedAt: new Date() } }); + }); + }); +}; + +Migrations.add({ + version: 213, + up() { + console.log('starting fix'); + Promise.await((async () => { + const options = { + projection: { rid: 1, u: 1, name: 1 }, + }; + const cursor = Subscriptions.find({ t: 'd' }, options).limit(100); + const total = await cursor.count(); + + console.log('total ->', total); + + // if number of subscription is low, we can go ahead and fix them all + if (total < 5) { // 1000) { + return updateSubscriptions(); + } + + // otherwise we'll first see if they're broken + const subs = await cursor.toArray(); + const subsTotal = subs.length; + console.log('subsTotal ->', subsTotal); + + const subsWithRoom = await Promise.all(subs.map(async (sub) => ({ + sub, + room: await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }), + }))); + + const wrongSubs = subsWithRoom + .filter(({ room }) => room && room.usernames && room.usernames.length > 0) + .filter(({ room, sub }) => { + const name = room.usernames + .filter((u) => u !== sub.u.username) + .sort() + .join(', ') || sub.u.username; + + return name !== sub.name; + }).length; + + + console.log('wrongSubs ->', wrongSubs); + + // if less then 10% of subscriptions are wrong, we're fine, doesn't need to do anything + if (wrongSubs / subsTotal < 0.1) { + return; + } + + return updateSubscriptions(); + })()); + console.log('fix done'); + }, +}); From dd815e8c291546b5b96a9648be5860cbed5b621c Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 12 Jan 2021 12:05:36 -0300 Subject: [PATCH 2/6] Fix migration --- server/startup/migrations/v213.js | 67 ++++++++++++++++--------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/server/startup/migrations/v213.js b/server/startup/migrations/v213.js index 21cb3df07f026..3eb45018a807c 100644 --- a/server/startup/migrations/v213.js +++ b/server/startup/migrations/v213.js @@ -3,38 +3,41 @@ import { Subscriptions, Rooms } from '../../../app/models/server/raw'; const batchSize = 2; -const updateSubscriptions = async (total, current) => { - return new Promise(async (resolve, reject) => { - Subscriptions - .find({ t: 'd' }, { projection: { rid: 1, u: 1 } }) - .limit(batchSize) - .forEach(async (sub) => { - console.log('sub', sub); - const room = await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }); - if (!room) { - console.log(`ROOM NOT FOUND: ${ sub.rid }`); - return; - } - - if (!room.usernames || room.usernames.length === 0) { - console.log(`NO USERNAMES: ${ sub.rid }`); - return; - } - - const name = room.usernames - .filter((u) => u !== sub.u.username) - .sort() - .join(', ') || sub.u.username; - if (!name) { - console.log(`NO NAME ${ sub._id }`); - return; - } - - console.log('fix', sub._id); - await Subscriptions.update({ _id: sub._id }, { $set: { name, _updatedAt: new Date() } }); - }); - }); -}; +const updateSubscriptions = async () => new Promise(async (resolve, reject) => { + Subscriptions + .find({ t: 'd' }, { projection: { rid: 1, u: 1 } }) + .limit(batchSize) + .forEach(async (sub) => { + console.log('sub', sub); + const room = await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }); + if (!room) { + console.log(`ROOM NOT FOUND: ${ sub.rid }`); + return; + } + + if (!room.usernames || room.usernames.length === 0) { + console.log(`NO USERNAMES: ${ sub.rid }`); + return; + } + + const name = room.usernames + .filter((u) => u !== sub.u.username) + .sort() + .join(', ') || sub.u.username; + if (!name) { + console.log(`NO NAME ${ sub._id }`); + return; + } + + console.log('fix', sub._id); + await Subscriptions.update({ _id: sub._id }, { $set: { name, _updatedAt: new Date() } }); + }, (error) => { + if (error) { + return reject(error); + } + resolve(); + }); +}); Migrations.add({ version: 213, From b23691a9d432601b09d0597b23cd476f9b57e2b6 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 12 Jan 2021 16:34:12 -0300 Subject: [PATCH 3/6] Finish migration --- server/startup/migrations/v213.js | 90 +++++++++++++++---------------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/server/startup/migrations/v213.js b/server/startup/migrations/v213.js index 3eb45018a807c..8b56f35a52cef 100644 --- a/server/startup/migrations/v213.js +++ b/server/startup/migrations/v213.js @@ -1,48 +1,52 @@ import { Migrations } from '../../../app/migrations'; import { Subscriptions, Rooms } from '../../../app/models/server/raw'; -const batchSize = 2; - -const updateSubscriptions = async () => new Promise(async (resolve, reject) => { - Subscriptions - .find({ t: 'd' }, { projection: { rid: 1, u: 1 } }) - .limit(batchSize) - .forEach(async (sub) => { - console.log('sub', sub); - const room = await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }); - if (!room) { - console.log(`ROOM NOT FOUND: ${ sub.rid }`); - return; - } - - if (!room.usernames || room.usernames.length === 0) { - console.log(`NO USERNAMES: ${ sub.rid }`); - return; - } - - const name = room.usernames - .filter((u) => u !== sub.u.username) - .sort() - .join(', ') || sub.u.username; - if (!name) { - console.log(`NO NAME ${ sub._id }`); - return; - } - - console.log('fix', sub._id); - await Subscriptions.update({ _id: sub._id }, { $set: { name, _updatedAt: new Date() } }); - }, (error) => { - if (error) { - return reject(error); - } - resolve(); +const updateSubscriptions = async () => { + const cursor = Subscriptions.find({ t: 'd' }, { projection: { rid: 1, u: 1 } }); + + let actions = []; + for await (const sub of cursor) { + const room = await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }); + if (!room) { + console.log(`[migration] room record not found: ${ sub.rid }`); + return; + } + + if (!room.usernames || room.usernames.length === 0) { + console.log(`[migration] room without usernames: ${ sub.rid }`); + return; + } + + const name = room.usernames + .filter((u) => u !== sub.u.username) + .sort() + .join(', ') || sub.u.username; + if (!name) { + console.log(`[migration] subscription without name ${ sub._id }`); + return; + } + + await Subscriptions.update({ _id: sub._id }, { $set: { name, _updatedAt: new Date() } }); + + actions.push({ + updateMany: { + filter: { _id: sub._id }, + update: { $set: { name, _updatedAt: new Date() } }, + }, }); -}); + if (actions.length === 1000) { + await Subscriptions.col.bulkWrite(actions, { ordered: false }); + actions = []; + } + } + if (actions.length) { + await Subscriptions.col.bulkWrite(actions, { ordered: false }); + } +}; Migrations.add({ version: 213, up() { - console.log('starting fix'); Promise.await((async () => { const options = { projection: { rid: 1, u: 1, name: 1 }, @@ -50,17 +54,14 @@ Migrations.add({ const cursor = Subscriptions.find({ t: 'd' }, options).limit(100); const total = await cursor.count(); - console.log('total ->', total); - // if number of subscription is low, we can go ahead and fix them all - if (total < 5) { // 1000) { + if (total < 1000) { return updateSubscriptions(); } // otherwise we'll first see if they're broken const subs = await cursor.toArray(); const subsTotal = subs.length; - console.log('subsTotal ->', subsTotal); const subsWithRoom = await Promise.all(subs.map(async (sub) => ({ sub, @@ -79,15 +80,12 @@ Migrations.add({ }).length; - console.log('wrongSubs ->', wrongSubs); - - // if less then 10% of subscriptions are wrong, we're fine, doesn't need to do anything - if (wrongSubs / subsTotal < 0.1) { + // if less then 5% of subscriptions are wrong, we're fine, doesn't need to do anything + if (wrongSubs / subsTotal < 0.05) { return; } return updateSubscriptions(); })()); - console.log('fix done'); }, }); From c01305c617cae1925ea435a29ff9292ce2aec970 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 12 Jan 2021 16:52:52 -0300 Subject: [PATCH 4/6] Remove wrong update --- server/startup/migrations/v213.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/startup/migrations/v213.js b/server/startup/migrations/v213.js index 8b56f35a52cef..7566e646b2683 100644 --- a/server/startup/migrations/v213.js +++ b/server/startup/migrations/v213.js @@ -26,8 +26,6 @@ const updateSubscriptions = async () => { return; } - await Subscriptions.update({ _id: sub._id }, { $set: { name, _updatedAt: new Date() } }); - actions.push({ updateMany: { filter: { _id: sub._id }, From cdae3bdd6bec528dd79e1b4becdfb7ce6ca17ff0 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 12 Jan 2021 17:07:27 -0300 Subject: [PATCH 5/6] Get least recent subscriptions --- server/startup/migrations/v213.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/startup/migrations/v213.js b/server/startup/migrations/v213.js index 7566e646b2683..c2df5816c3133 100644 --- a/server/startup/migrations/v213.js +++ b/server/startup/migrations/v213.js @@ -49,7 +49,7 @@ Migrations.add({ const options = { projection: { rid: 1, u: 1, name: 1 }, }; - const cursor = Subscriptions.find({ t: 'd' }, options).limit(100); + const cursor = Subscriptions.find({ t: 'd' }, options).sort({ _updatedAt: 1 }).limit(100); const total = await cursor.count(); // if number of subscription is low, we can go ahead and fix them all From a101cb16cf1b3ac2bb3165d5e58af8f3db2edbf9 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Wed, 13 Jan 2021 00:06:21 -0300 Subject: [PATCH 6/6] Use continue --- server/startup/migrations/v213.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/startup/migrations/v213.js b/server/startup/migrations/v213.js index c2df5816c3133..7752f0d047cd0 100644 --- a/server/startup/migrations/v213.js +++ b/server/startup/migrations/v213.js @@ -9,12 +9,12 @@ const updateSubscriptions = async () => { const room = await Rooms.findOne({ _id: sub.rid }, { projection: { usernames: 1 } }); if (!room) { console.log(`[migration] room record not found: ${ sub.rid }`); - return; + continue; } if (!room.usernames || room.usernames.length === 0) { console.log(`[migration] room without usernames: ${ sub.rid }`); - return; + continue; } const name = room.usernames @@ -23,7 +23,7 @@ const updateSubscriptions = async () => { .join(', ') || sub.u.username; if (!name) { console.log(`[migration] subscription without name ${ sub._id }`); - return; + continue; } actions.push({