From e8b516fd97a1b6a8c24e8f187e6dc5058713568e Mon Sep 17 00:00:00 2001 From: LorenzoJokhan Date: Thu, 2 Mar 2023 16:28:12 +0100 Subject: [PATCH 1/5] Rework classes to anonymize users in the background --- src/models/Site.js | 60 +++++++++++++++++++++++++++--------------- src/routes/api/site.js | 38 ++++++++------------------ 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/src/models/Site.js b/src/models/Site.js index f9cfb8e0..2bc727d5 100755 --- a/src/models/Site.js +++ b/src/models/Site.js @@ -4,6 +4,7 @@ const moment = require('moment'); const apiConfig = require('config'); const OAuthApi = require('../services/oauth-api'); const userHasRole = require('../lib/sequelize-authorization/lib/hasRole'); +const tasks = require('../../../auth/db/tasks'); module.exports = function (db, sequelize, DataTypes) { @@ -912,12 +913,10 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o } Site.prototype.willAnonymizeAllUsers = async function () { - let self = this; let result = {}; try { - if (!self.id) throw Error('Site not found'); if (!self.config.project.projectHasEnded) throw Error('Cannot anonymize users on an active site - first set the project-has-ended parameter'); @@ -930,42 +929,61 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o // extract externalUserIds result.externalUserIds = result.users.filter( user => user.externalUserId ).map( user => user.externalUserId ); + console.log({result}); } catch (err) { console.log(err); throw err; } return result; - } - Site.prototype.doAnonymizeAllUsers = async function () { - + Site.prototype.doAnonymizeAllUsers = async function (usersToAnonymize, externalUserIds, req) { // anonymize all users for this site let self = this; - let result; - + const amountOfUsersPerSecond = 50; try { - - result = await self.willAnonymizeAllUsers(); - - let users = [ ...result.users ] - - // anonymize users - for (const user of users) { - user.site = self; - let res = await user.doAnonymize(); - user.site = null; + let task = await tasks.save(null, { + userCountToAnonymize: usersToAnonymize.length, + anonymizedUsers: 0 + }); + let taskId = req.taskId = task.taskId; + + // Anonymize users + for (const user of usersToAnonymize) { + await new Promise((resolve, reject) => { + setTimeout(async function() { + user.site = self; + let res = await user.doAnonymize(); + user.site = null; + task.anonymizedUsers++; + }, 1000 / amountOfUsersPerSecond) + }) + .then(result => { + task.anonymizedUsers++; + return tasks.save(taskId, task); + }) + .then(result => resolve() ) + .catch(function (err) { + throw err; + }); } + for (let externalUserId of externalUserIds) { + let users = await db.User.findAll({ where: { externalUserId } }); + if (users.length == 0) { + // no api users left for this oauth user, so remove the oauth user + let which = req.query.useOauth || 'default'; + let siteConfig = req.site && merge({}, req.site.config, { id: req.site.id }); + if (req.params.willOrDo == 'do') { + await OAuthApi.deleteUser({ siteConfig, which, userData: { id: externalUserId }}) + } + } + } } catch (err) { console.log(err); throw err; } - - result.message = 'Ok'; - return result; - } Site.prototype.isVoteActive = function () { diff --git a/src/routes/api/site.js b/src/routes/api/site.js index 77f1d19e..ce76c1fc 100755 --- a/src/routes/api/site.js +++ b/src/routes/api/site.js @@ -282,40 +282,24 @@ router.route('/:siteId(\\d+)/:willOrDo(will|do)-anonymize-all-users') if ( !found ) throw new Error('Site not found'); req.results = found; req.site = req.results; // middleware expects this to exist - next(); - return null; + next(); }) .catch(next); }) .put(async function (req, res, next) { try { + const result = await req.site.willAnonymizeAllUsers(); + req.results = result; + console.log({result}); if (req.params.willOrDo == 'do') { - req.results = await req.site.doAnonymizeAllUsers(); - } else { - req.results = await req.site.willAnonymizeAllUsers(); - } + result.message = 'Ok'; + req.site.doAnonymizeAllUsers( + [...result.users], + [...result.externalUserIds], + req + ); + } next(); - return null; - } catch (err) { - return next(err); - } - }) - .put(async function (req, res, next) { - try { - if (req.results.externalUserIds.length == 0) return next(); - for (let externalUserId of req.results.externalUserIds) { - let users = await db.User.findAll({ where: { externalUserId } }); - if (users.length == 0) { - // no api users left for this oauth user, so remove the oauth user - let which = req.query.useOauth || 'default'; - let siteConfig = req.site && merge({}, req.site.config, { id: req.site.id }); - if (req.params.willOrDo == 'do') { - let result = await OAuthApi.deleteUser({ siteConfig, which, userData: { id: externalUserId }}) - } - } - } - next(); - return null; } catch (err) { return next(err); } From 953e3c78d74c55b17adf729e27b35421dd9d4937 Mon Sep 17 00:00:00 2001 From: LorenzoJokhan Date: Mon, 6 Mar 2023 10:04:48 +0100 Subject: [PATCH 2/5] remove console.log statement --- src/routes/api/site.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/routes/api/site.js b/src/routes/api/site.js index ce76c1fc..a8f483d2 100755 --- a/src/routes/api/site.js +++ b/src/routes/api/site.js @@ -290,7 +290,6 @@ router.route('/:siteId(\\d+)/:willOrDo(will|do)-anonymize-all-users') try { const result = await req.site.willAnonymizeAllUsers(); req.results = result; - console.log({result}); if (req.params.willOrDo == 'do') { result.message = 'Ok'; req.site.doAnonymizeAllUsers( From d1cee0ffac12a1e7a80a63f28cd6461cd6381c4a Mon Sep 17 00:00:00 2001 From: LorenzoJokhan Date: Tue, 7 Mar 2023 09:34:35 +0100 Subject: [PATCH 3/5] Remove task code which does not really seem usefull in any way --- src/models/Site.js | 16 +--------------- src/routes/api/site.js | 3 ++- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/models/Site.js b/src/models/Site.js index 2bc727d5..e392b822 100755 --- a/src/models/Site.js +++ b/src/models/Site.js @@ -4,7 +4,6 @@ const moment = require('moment'); const apiConfig = require('config'); const OAuthApi = require('../services/oauth-api'); const userHasRole = require('../lib/sequelize-authorization/lib/hasRole'); -const tasks = require('../../../auth/db/tasks'); module.exports = function (db, sequelize, DataTypes) { @@ -928,8 +927,6 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o // extract externalUserIds result.externalUserIds = result.users.filter( user => user.externalUserId ).map( user => user.externalUserId ); - - console.log({result}); } catch (err) { console.log(err); throw err; @@ -938,17 +935,11 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o return result; } - Site.prototype.doAnonymizeAllUsers = async function (usersToAnonymize, externalUserIds, req) { + Site.prototype.doAnonymizeAllUsers = async function (usersToAnonymize, externalUserIds) { // anonymize all users for this site let self = this; const amountOfUsersPerSecond = 50; try { - let task = await tasks.save(null, { - userCountToAnonymize: usersToAnonymize.length, - anonymizedUsers: 0 - }); - let taskId = req.taskId = task.taskId; - // Anonymize users for (const user of usersToAnonymize) { await new Promise((resolve, reject) => { @@ -956,13 +947,8 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o user.site = self; let res = await user.doAnonymize(); user.site = null; - task.anonymizedUsers++; }, 1000 / amountOfUsersPerSecond) }) - .then(result => { - task.anonymizedUsers++; - return tasks.save(taskId, task); - }) .then(result => resolve() ) .catch(function (err) { throw err; diff --git a/src/routes/api/site.js b/src/routes/api/site.js index a8f483d2..adf518b6 100755 --- a/src/routes/api/site.js +++ b/src/routes/api/site.js @@ -11,6 +11,7 @@ const oauthClients = require('../../middleware/oauth-clients'); const checkHostStatus = require('../../services/checkHostStatus') const OAuthApi = require('../../services/oauth-api'); const sitesWithIssues = require('../../services/sites-with-issues'); +const tasks = require('../../../auth/db/tasks'); let router = express.Router({mergeParams: true}); @@ -292,10 +293,10 @@ router.route('/:siteId(\\d+)/:willOrDo(will|do)-anonymize-all-users') req.results = result; if (req.params.willOrDo == 'do') { result.message = 'Ok'; + req.site.doAnonymizeAllUsers( [...result.users], [...result.externalUserIds], - req ); } next(); From 481e5e95183f24fe43bf5a3bd573159aff838de0 Mon Sep 17 00:00:00 2001 From: LorenzoJokhan Date: Tue, 7 Mar 2023 09:51:43 +0100 Subject: [PATCH 4/5] Remove the do check inside method doAnonymizeAllUsers, it will always be in 'do' replace req param with self --- src/models/Site.js | 10 ++++------ src/routes/api/site.js | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/models/Site.js b/src/models/Site.js index e392b822..842155f0 100755 --- a/src/models/Site.js +++ b/src/models/Site.js @@ -935,13 +935,14 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o return result; } - Site.prototype.doAnonymizeAllUsers = async function (usersToAnonymize, externalUserIds) { + Site.prototype.doAnonymizeAllUsers = async function (usersToAnonymize, externalUserIds, useOauth='default') { // anonymize all users for this site let self = this; const amountOfUsersPerSecond = 50; try { // Anonymize users for (const user of usersToAnonymize) { + console.log({user}); await new Promise((resolve, reject) => { setTimeout(async function() { user.site = self; @@ -959,11 +960,8 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o let users = await db.User.findAll({ where: { externalUserId } }); if (users.length == 0) { // no api users left for this oauth user, so remove the oauth user - let which = req.query.useOauth || 'default'; - let siteConfig = req.site && merge({}, req.site.config, { id: req.site.id }); - if (req.params.willOrDo == 'do') { - await OAuthApi.deleteUser({ siteConfig, which, userData: { id: externalUserId }}) - } + let siteConfig = self && merge({}, self.config, { id: self.id }); + await OAuthApi.deleteUser({ siteConfig, useOauth, userData: { id: externalUserId }}) } } } catch (err) { diff --git a/src/routes/api/site.js b/src/routes/api/site.js index adf518b6..9761ed1b 100755 --- a/src/routes/api/site.js +++ b/src/routes/api/site.js @@ -11,7 +11,6 @@ const oauthClients = require('../../middleware/oauth-clients'); const checkHostStatus = require('../../services/checkHostStatus') const OAuthApi = require('../../services/oauth-api'); const sitesWithIssues = require('../../services/sites-with-issues'); -const tasks = require('../../../auth/db/tasks'); let router = express.Router({mergeParams: true}); @@ -293,10 +292,12 @@ router.route('/:siteId(\\d+)/:willOrDo(will|do)-anonymize-all-users') req.results = result; if (req.params.willOrDo == 'do') { result.message = 'Ok'; - + req.site.doAnonymizeAllUsers( [...result.users], [...result.externalUserIds], + req.query.useOauth + ); } next(); From 9e807f520c01c49be47254bef361cade62e5d4f8 Mon Sep 17 00:00:00 2001 From: LorenzoJokhan Date: Tue, 7 Mar 2023 09:53:39 +0100 Subject: [PATCH 5/5] Remove dangling console log statement --- src/models/Site.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/models/Site.js b/src/models/Site.js index 842155f0..dfd84efb 100755 --- a/src/models/Site.js +++ b/src/models/Site.js @@ -942,7 +942,6 @@ Wil je dit liever niet? Dan hoef je alleen een keer in te loggen op de website o try { // Anonymize users for (const user of usersToAnonymize) { - console.log({user}); await new Promise((resolve, reject) => { setTimeout(async function() { user.site = self;