Merge pull request #426 from matchID-project/feat/jobs-priorisation

🤴🏾 Jobs priorisation

.github/workflows/dockerimage.yml

@@ -28,7 +28,7 @@ jobs:
       - name: 🐋 Save docker image
         run: make docker-save
       - name: ⬆ Upload docker image
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: deces-backend
           path: deces-backend.tar
@@ -40,7 +40,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: ⬇ Download docker image
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: deces-backend
       - name: 🐳 Load docker image
@@ -60,7 +60,7 @@ jobs:
         run: |
           make backend-perf-clinic-stop
       - name: 📦 Upload results as job artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: match-results
           path: |

Makefile

@@ -39,6 +39,7 @@ export PORT=8084
 export BACKEND=${APP_PATH}/backend
 export BACKEND_PORT=8080
 export BACKEND_HOST=backend
+export APP_URL?=https://${APP_DNS}
 export API_URL?=localhost:${PORT}
 export API_EMAIL?[email protected]
 export API_SSL?=1

backend/src/controllers/bulk.controller.ts

@@ -147,6 +147,7 @@ export class BulkController extends Controller {
       options.candidateNumber = options.candidateNumber || 1;
       options.pruneScore = options.pruneScore !== undefined ? parseFloat(options.pruneScore) : undefined;
       options.mapField = {};
+      options.tmpfilePersistence = Math.max(60000, options.tmpfilePersistence || process.env.BACKEND_TMPFILE_PERSISTENCE);
       validFields.forEach(key => options.mapField[options[key] || key] = key );
       return await csvHandle(request, options)
     } else {

backend/src/controllers/job.controller.ts

@@ -1,5 +1,6 @@
+import express from 'express';
 import { Queue, JobType } from 'bullmq';
-import { Controller, Get, Route, Tags, Path, Query, Security } from 'tsoa';
+import { Controller, Get, Route, Tags, Path, Query, Request, Security } from 'tsoa';
 
 /**
  * @swagger
@@ -38,29 +39,41 @@ export class JobsController extends Controller {
    * @param queueName Name of queue
    * @param jobsType Jobs type or Id
    */
-  @Security("jwt", ["admin"])
+  @Security("jwt", ["user"])
   @Tags('Jobs')
   @Get('{queueName}')
   public async getJobs(
-
+    @Request() request: express.Request,
     @Path() queueName: 'jobs',
     @Query() jobId?: string,
     @Query() jobsType?: JobType,
   ): Promise<any> {
+    const scopes = (request as any).user && (request as any).user.scopes
+    const user = (request as any).user && (request as any).user.user
     const jobQueue = new Queue(queueName,  {
       connection: {
         host: 'redis'
       }
     });
-    const jobsTypeList: JobType[] = ['completed', 'failed', 'active', 'delayed', 'waiting', 'waiting-children', 'paused', 'repeat', 'wait']
+    const jobsTypeList: JobType[] = ['completed', 'failed', 'active', 'delayed', 'waiting', 'waiting-children', 'paused', 'repeat', 'wait', 'prioritized']
     if (jobsTypeList.includes(jobsType)) {
       const jobs = await jobQueue.getJobs(jobsType);
-      jobs.forEach((j: any) => delete j.data.randomKey)
-      return { jobs };
+      let jobsUser;
+      if (scopes.includes('admin')) {
+        jobsUser = jobs;
+        // admin should not see randomKey
+        jobsUser.forEach((j: any) => j.data.randomKey = undefined)
+      } else {
+        // user only see their jobs, and get their randomKey to be able to get their files
+        jobsUser = jobs.filter((job: any) => job.data.user === user);
+      }
+      return { jobs: jobsUser };
     } else if (jobId) {
       const job = await jobQueue.getJob(jobId)
-      delete job.data.randomKey
       if (job) {
+        if (scopes.includes('admin')) {
+          job.data.randomKey = undefined;
+        }
         return { job };
       } else {
         return { msg: 'job not found' };
@@ -69,11 +82,19 @@ export class JobsController extends Controller {
       let jobs:any = []
       for (const jobType of jobsTypeList) {
         const jobsTmp = await jobQueue.getJobs(jobType);
-        jobsTmp.forEach((j: any) => {
-          j.status = jobType
-          delete j.data.randomKey
+        let jobsTmpUser;
+        if (scopes.includes('admin')) {
+          jobsTmpUser = jobsTmp;
+        } else {
+          jobsTmpUser = jobsTmp.filter((job: any) => job.data.user === user);
+        }
+        jobsTmpUser.forEach((j: any) => {
+          j.status = jobType;
+          if (scopes.includes('admin')) {
+            j.data.randomKey = undefined;
+          }
         });
-        jobs = [...jobs, ...jobsTmp]
+        jobs = [...jobs, ...jobsTmpUser]
       }
       return { jobs };
     }

backend/src/mail.ts

@@ -105,6 +105,32 @@ export const validateOTP = (email:string,otp:string): boolean => {
     return false;
 }
 
+export const sendJobUpdate = async (email:string, content: string, jobId: string): Promise<boolean> => {
+    try {
+        const message: any = {
+            from: process.env.API_EMAIL,
+            to: `${email}`,
+        }
+        message.subject = `Traitement sur un fichier - ${process.env.APP_DNS}`;
+        message.text = `${content ? 'Traitement fichier: ' + content : ''}\nVous pouvez consulter le status du traitement <a href="${process.env.APP_URL}/link?job=${jobId}"></a>ici.<br>`
+        message.html = `<html style="font-family:Helvetica;">
+              <h4> Traitement d'un fichier </h4>
+              Vous avez lancé une tache d'appariement,<br>
+              <br>
+              ${content ? '<br>' + content + '<br>' : ''}<br>
+              Vous pouvez consulter le status du traitement <a href="${process.env.APP_URL}/link?job=${jobId}">en utilisant ce lien</a>.<br>
+              <br>
+              l'équipe matchID
+              </html>
+              `
+        await transporter.sendMail(message);
+        return true;
+    } catch (err) {
+        return false;
+    }
+}
+
+
 export const sendUpdateConfirmation = async (email:string, status: ReviewStatus, rejectMsg: string, id: string): Promise<boolean> => {
     try {
         const message: any = {
@@ -116,9 +142,9 @@ export const sendUpdateConfirmation = async (email:string, status: ReviewStatus,
             message.attachment = { data: `<html style="font-family:Helvetica;">
             <h4> Merci de votre contibution !</h4>
             Votre proposition de correction a été acceptée.<br>
-            Retrouvez <a href="https://${process.env.APP_DNS}/id/${id}"> la fiche modifiée </a>.<br>
+            Retrouvez <a href="${process.env.APP_URL}/id/${id}"> la fiche modifiée </a>.<br>
             <br>
-            Vous pouvez à tout moment <a href="https://${process.env.APP_DNS}/edits">revenir sur vos contributions</a>.<br>
+            Vous pouvez à tout moment <a href="${process.env.APP_URL}/edits">revenir sur vos contributions</a>.<br>
             <br>
             l'équipe matchID
             </html>
@@ -130,7 +156,7 @@ export const sendUpdateConfirmation = async (email:string, status: ReviewStatus,
             <br>
             Néanmoins les éléments fournis ne nous ont pas permis de retenir votre proposition à ce stade.<br>
             ${rejectMsg ? '<br>' + rejectMsg + '<br>' : ''}<br>
-            Vous pourrez de nouveau soumettre une nouvelle proposition sur la fiche: <a href="https://${process.env.APP_DNS}/edits#${id}"></a>.<br>
+            Vous pourrez de nouveau soumettre une nouvelle proposition sur la fiche <a href="${process.env.APP_URL}/edits#${id}">ici</a>.<br>
             <br>
             l'équipe matchID
             </html>