Merge pull request #930 from matchID-project/feat/jobs-priorisation

🤴🏾 Jobs priorisation UI
matchID-project · Oct 14, 2024 · fc45673 · fc45673
2 parents 9007a67 + ce8c767
commit fc45673
Show file tree

Hide file tree

Showing 12 changed files with 2,268 additions and 4,977 deletions.
diff --git a/Makefile b/Makefile
@@ -31,7 +31,7 @@ export APP = deces-ui
 export DATASET=fichier-des-personnes-decedees
 export APP_GROUP = matchID
 export APP_PATH := $(shell pwd)
-export APP_DNS=deces.matchid.io
+export APP_DNS?=deces.matchid.io
 export API_EMAIL?[email protected]
 export FRONTEND := ${APP_PATH}
 export FRONTEND_DEV_HOST = frontend-development
@@ -110,7 +110,7 @@ export BACKEND_APP=${GIT_BACKEND}
 export GIT_BACKEND_BRANCH ?= dev
 export GIT_ROOT = https://github.com/matchid-project
 export GIT_TOOLS = tools
-export API_URL?=${APP_DNS}
+export APP_URL?=https://${APP_DNS}
 export API_SSL?=1
 export APP_NODES=1
 export KUBE_NAMESPACE:=$(shell echo -n ${APP_GROUP}-${APP}-${GIT_BRANCH} | tr '[:upper:]' '[:lower:]' | tr '_/' '-')
@@ -279,7 +279,7 @@ backend-config:
 backend-dev: backend-config
 	@echo docker-compose up backend dev
 	@${MAKE} -C ${APP_PATH}/${GIT_BACKEND} dev DC_NETWORK=${DC_NETWORK} GIT_BRANCH=${GIT_BACKEND_BRANCH}\
-		API_URL=${API_URL} API_EMAIL=${API_EMAIL} API_SSL=${API_SSL}\
+		APP_URL=http://localhost:${PORT} API_URL=http://localhost:${PORT} API_EMAIL=${API_EMAIL} API_SSL=${API_SSL}\
 		BACKEND_JOB_CONCURRENCY=${BACKEND_JOB_CONCURRENCY} BACKEND_CHUNK_CONCURRENCY=${BACKEND_CHUNK_CONCURRENCY}\
 		BACKEND_TOKEN_USER=${BACKEND_TOKEN_USER} BACKEND_TOKEN_KEY=${BACKEND_TOKEN_KEY} BACKEND_TOKEN_PASSWORD=${BACKEND_TOKEN_PASSWORD}\
 		BACKEND_TMP_MAX=${BACKEND_TMP_MAX} BACKEND_TMP_DURATION=${BACKEND_TMP_DURATION} BACKEND_TMP_WINDOW=${BACKEND_TMP_WINDOW}
@@ -297,7 +297,7 @@ backend-docker-check: backend-config
 backend: backend-config backend-docker-check proofs-mount elasticsearch-index-readiness
 	@BACKEND_APP_VERSION=$(shell cd ${APP_PATH}/${GIT_BACKEND} && git describe --tags);\
 	${MAKE} -C ${APP_PATH}/${GIT_BACKEND} backend-start APP=deces-backend DC_NETWORK=${DC_NETWORK} APP_VERSION=$$BACKEND_APP_VERSION GIT_BRANCH=${GIT_BACKEND_BRANCH}\
-		API_URL=${API_URL} API_EMAIL=${API_EMAIL} API_SSL=${API_SSL}\
+		APP_URL=${APP_URL} API_URL=${API_URL} API_EMAIL=${API_EMAIL} API_SSL=${API_SSL}\
                 BACKEND_JOB_CONCURRENCY=${BACKEND_JOB_CONCURRENCY} BACKEND_CHUNK_CONCURRENCY=${BACKEND_CHUNK_CONCURRENCY}\
                 BACKEND_TOKEN_USER=${BACKEND_TOKEN_USER} BACKEND_TOKEN_KEY=${BACKEND_TOKEN_KEY} BACKEND_TOKEN_PASSWORD=${BACKEND_TOKEN_PASSWORD}\
                 BACKEND_TMP_MAX=${BACKEND_TMP_MAX} BACKEND_TMP_DURATION=${BACKEND_TMP_DURATION} BACKEND_TMP_WINDOW=${BACKEND_TMP_WINDOW}

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -32,7 +32,7 @@
     "shake.js": "^1.2.2",
     "sirv-cli": "^2.0.2",
     "svelte": "^3.55.0",
-    "svelte-scrollto": "0.2.0",
+    "svelte-scrolling": "1.4.0",
     "web-streams-polyfill": "^3.2.1",
     "yamljs": "^0.3.0"
   }

diff --git a/src/components/tools/jobs.js b/src/components/tools/jobs.js
@@ -0,0 +1,68 @@
+export const validColumns = [
+    'lastName',
+    'legalName',
+    'sex',
+    'birthDate',
+    'birthCity',
+    'birthDepartment',
+    'birthCountry',
+    'deathDate',
+    'deathCity',
+    'deathDepartment',
+    'deathCountry',
+    'lastSeenAliveDate'
+];
+
+export const getJobsData = async (accessToken) => {
+    const headers = {
+        headers: {
+            Authorization: `Bearer ${accessToken}`
+        }
+    }
+    let response = await fetch('__BACKEND_PROXY_PATH__/queue/jobs', headers);
+    const list = (await response.json()).jobs || [];
+    return list
+}
+
+export const getJobsFilteredData = async (accessToken) => {
+    const list = await getJobsData(accessToken);
+    const tmpJobs = [];
+    list.forEach(j => {
+        const delay = (
+            (j.finishedOn ? j.finishedOn : Math.floor(Date.now()))
+            - j.processedOn
+        ) / 1000;
+        const progress = j.status && j.status === "completed" ?
+            "100" : j.progress && j.progress.percentage ?
+                Math.round(j.progress.percentage) : 0
+        tmpJobs.push({
+            rows: j.data.totalRows,
+            id: j.id,
+            user: j.data && j.data.user,
+            date: j.timestamp,
+            status: j.status,
+            waiting_time: j.processedOn && (j.processedOn - j.timestamp) / 1000,
+            processing_time: delay,
+            columns: validColumns.filter(c => j.data && j.data[c]),
+            processing_rate: j.processedOn && Math.floor((progress / 100) * (j.data.totalRows / delay)),
+            finishedOnTime: j.finishedOn ? dateTostr(new Date(j.finishedOn)) : 'en cours',
+            deletionTime: j.status === 'completed' ? dateTostr(new Date(j.finishedOn + (j.data.tmpfilePersistence || 3600000))) : (j.status === 'failed' ? dateTostr(new Date(j.finishedOn)) : 'en cours'),
+            link: (['active', 'created', 'waiting', 'wait'].includes(j.status) || j.status === 'completed' && j.finishedOn && (Date.now() < new Date(j.finishedOn + (j.data.tmpfilePersistence || 3600000)))) ? `/link?job=${j.data.randomKey}` : undefined,
+            progress: progress
+        })});
+    return tmpJobs.sort((a,b) => (b.date - a.date)).map(j => {
+        j.date= dateTostr(new Date(j.date));
+        return j;
+    });
+}
+
+
+export const getJobData = async (accessToken, linkJob) => {
+    const jobs = await getJobsData(accessToken);
+    return jobs.find(j => j.data && j.data.randomKey && j.data.randomKey === linkJob);
+}
+
+const dateTostr = (_date) => {
+    const pad = (num) => num.toString().padStart(2, '0');
+    return `${_date.getFullYear()}-${pad(_date.getMonth() + 1)}-${pad(_date.getDate())} ${pad(_date.getHours())}:${pad(_date.getMinutes())}`;
+}
diff --git a/src/components/tools/routes.js b/src/components/tools/routes.js
@@ -41,7 +41,7 @@ export const routes = {
     },
     '/about': { component: Default, props: {title: 'À propos'}, title: 'À propos', desc: 'À propos de matchID: conditions légales données et code opensource, foire aux questions, documentation de l\'API, statistiques de consultation, ...'},
     '/link': { component: Link, title: 'Appariement', desc: 'matchID permet l\'appariement de votre fichier client, jusque 1 million d\'enregistrements, au fichier des décès de l`INSEE' },
-    '/jobs': { auth: true, component: LinkJobs, title: 'Traitements', desc: 'page d\'administration des traitements' },
+    '/jobs': { component: LinkJobs, title: 'Traitements', desc: 'page d\'administration des traitements' },
     '/edits': { auth: true, component: EditsList, title: 'Modifications', desc: 'page d\'administration des modifications' },
     '/stats': { component: Stats, title: 'statistiques de fréquentation', desc: 'statistiques de consultation de deces.matchid.io: visiteurs uniques, appels API, ... historique depuis début 2020' },
     '/explain': { component: AlgoDetails, title: 'Explication de l\'algorithme', desc: 'détails sur l\'algorithme d\'appariement' },

diff --git a/src/components/views/EditsList.svelte b/src/components/views/EditsList.svelte
@@ -112,7 +112,7 @@
 
 <script>
     import { onMount } from 'svelte';
-    import * as animateScroll from 'svelte-scrollto';
+    import { scrollTo } from 'svelte-scrolling'
     import ResultCard from './ResultCard.svelte';
     import Icon from './Icon.svelte';
     import StatsTile from './StatsTile.svelte';
@@ -152,7 +152,7 @@
             location.hash = id;
             $route.hash = location.hash;
             setTimeout(() => {
-                animateScroll.scrollTo({element: `#${id}`, delay: 400});
+                scrollTo({ref: `#${id}`, duration: 400});
             }, 400);
         };
     };
@@ -168,4 +168,4 @@
     td {
         vertical-align: middle;
     }
-</style>
+</style>
diff --git a/src/components/views/Info.svelte b/src/components/views/Info.svelte
@@ -60,7 +60,7 @@
 
 <script>
   import { onMount } from 'svelte';
-  import * as animateScroll from 'svelte-scrollto';
+  import { scrollTo } from 'svelte-scrolling'
   import { slide } from 'svelte/transition';
   import { version, themeDnum, route } from '../tools/stores.js';
   import Icon from './Icon.svelte';
@@ -499,7 +499,7 @@
       $route.hash = location.hash;
       if(navigator.userAgent.toLowerCase().indexOf('firefox') > -1){
         setTimeout(() => {
-          animateScroll.scrollTo({element: `#${id}`, duration: 400});
+          scrollTo({ref: `#${id}`, duration: 400});
         }, 400);
       };
     };

diff --git a/src/components/views/Link.svelte b/src/components/views/Link.svelte
@@ -271,7 +271,7 @@
 
     $: steps[3].body = LinkCheck;
 
-    const reset = async () => {
+    const reset = async (full = false) => {
         await clearAll();
         steps[0].label = step0Label;
         steps[0].error = false;
@@ -288,7 +288,13 @@
         $linkOptions.csv = undefined;
         $linkSourceHeader = undefined;
         $linkSourceHeaderTypes = undefined;
-        $linkJob = undefined;
+        if (full) {
+            $linkJob = undefined;
+            // clear job param from url
+            const url = new URL(window.location.href);
+            url.searchParams.delete('job');
+            window.history.replaceState({}, document.title, url);
+        }
         $linkCompleteResults = undefined;
         $linkResults = undefined;
         $linkValidations = undefined;
@@ -308,9 +314,15 @@
         await useLocalSync(linkResults, 'linkResults');
         await useLocalSync(linkJob, 'linkJob');
         $linkWaiter = false;
-        if (!$linkMapping || !$linkFileName || !$linkOptions.csv || !$linkSourceHeader || !$linkJob) {
+        const params = new URLSearchParams(window.location.search);
+        const linkJobParam = params.get('job');
+        if (linkJobParam && linkJobParam !== $linkJob) {
+            await reset();
+            $linkJob = linkJobParam;
+            console.log('Set linkJob from url', $linkJob);
+        } else if ($linkJob === 'failed' || !$linkMapping || !$linkFileName || !$linkOptions.csv || !$linkSourceHeader || !$linkJob) {
             console.log('reset');
-            reset();
+            await reset(true);
         }
     })
 

diff --git a/src/components/views/LinkConfigureOptions.svelte b/src/components/views/LinkConfigureOptions.svelte
@@ -96,6 +96,7 @@
                 quote: {size:3,label: "Délimiteur", options:[['aucun', undefined],['guillemets doubles (")','"'],["guillemets simples (')","'"]]},
                 skipLines: {label: "Lignes sautées:", options: [['aucune',0],[1,1],[2,2],[3,3],[4,4],[5,5]]},
                 dateFormat: {size: 3,label: "Format des dates", options:[['dd/MM/yyyy','dd/MM/yyyy'],['yyyy-MM-dd','yyyy-MM-dd'],['yyyyMMdd','yyyyMMdd'],['ddMMyyyy','ddMMyyyy'],['dd-MM-yyyy','dd-MM-yyyy'],['yyyy/MM/dd']]},
+                tmpfilePersistence: {size: 2,label: "Temps de persistence du fichier", options:[['1 minute','60000'],['1 heure','3600000'],['4 heures','14400000'],['8 heures','28800000']]},
             }
         },
         api: {

diff --git a/src/components/views/LinkJob.svelte b/src/components/views/LinkJob.svelte
@@ -58,9 +58,10 @@
     export let error=false;
 
     import { accessToken, linkWaiter, linkMapping, linkFile, linkFileSize, linkJob, linkStep,
-        linkCompleteResults, linkResults, linkOptions,
+        linkCompleteResults, linkResults, linkOptions, linkFileName, linkSourceHeader,
         linkValidations, themeDnum
     } from '../tools/stores.js';
+    import { validColumns, getJobData } from '../tools/jobs.js';
     let progressUpload = 0;
     let progressJob = 0;
     let progressQueue = 0;
@@ -141,13 +142,16 @@
 
     $: axiosUploadConfig.headers = { Authorization: `Bearer ${$accessToken}` }
 
+    const headers = {};
+
     const axiosDownloadConfig = {
         onDownloadProgress: (progressEvent) => {
-            progressDownload = progressEvent.currentTarget.response.length * 100 / ($linkFileSize * upDownRatio);
-        }
+            progressDownload = progressEvent && progressEvent.currentTarget && progressEvent.currentTarget.response && progressEvent.currentTarget.response.length ? progressEvent.currentTarget.response.length * 100 / (($linkFileSize || 1000) * upDownRatio) : 0;
+        },
+        headers: headers
     };
 
-    $: axiosDownloadConfig.headers = { Authorization: `Bearer ${$accessToken}` }
+    $: headers.Authorization = `Bearer ${$accessToken}`;
 
     const upload = async () =>  {
         progressUpload = 0;
@@ -158,6 +162,7 @@
         formData.append('candidateNumber', $linkOptions.api.candidateNumber);
         formData.append('pruneScore', $linkOptions.api.pruneScore);
         formData.append('dateFormat', $linkOptions.csv.dateFormat || 'dd/MM/yyyy');
+        formData.append('tmpfilePersistence', $linkOptions.csv.tmpfilePersistence);
         Object.keys($linkMapping && $linkMapping.reverse).map(k => formData.append(k,$linkMapping.reverse[k]));
         if ($linkOptions.csv.type === 'gedcom') {
             formData.append('csv', new Blob([$linkOptions.csv.csv], {type: 'text/csv;charset=utf-8;'}));
@@ -181,37 +186,65 @@
     }
 
     const watchJob = async () =>  {
-        let res
-        if ($linkJob && !error) {
+        let res;
+        if ($linkJob && $linkJob !== 'failed' && !error) {
+            let res;
             try {
                 res = await axios.get(`__BACKEND_PROXY_PATH__/search/csv/${$linkJob}`, axiosDownloadConfig);
-                if (typeof(res.data) !== 'string') {
-                    if ((res.data.status === 'failed') || (res.data.msg === "job doesn't exists")) {
-                        error = `${res.data && res.data.msg || 'erreur inconnue'}`;
-                    } else {
-                        if (res.data.status === 'active') {
-                            waiting = null;
-                            if (res.data.progress && res.data.progress.percentage) {
-                                progressJob = res.data.progress.percentage;
-                            } else {
-                                progressJob = 0;
-                            }
+            } catch(err) {;
+                error = formatError(err);
+            }
+            if (res &&typeof(res.data) !== 'string') {
+                if ((res.data.status === 'failed') || (res.data.msg === "job doesn't exists")) {
+                    error = `${res.data && res.data.msg || 'erreur inconnue'}`;
+                } else {
+                    if (res.data.status === 'active') {
+                        waiting = null;
+                        if (res.data.progress && res.data.progress.percentage) {
+                            progressJob = res.data.progress.percentage;
                         } else {
-                            waiting = true;
-                            queueSize = res.data.remainingRowsWaiting + res.data.remainingRowsActive;
+                            progressJob = 0;
                         }
+                    } else if (res.data.msg === 'Job succeeded but results expired') {
+                        error = 'Les résultats ont expiré, merci de relancer le traitement';
+                    } else {
+                        waiting = true;
+                        queueSize = res.data.remainingRowsWaiting + res.data.remainingRowsActive;
                     }
-                } else {
-                    progressJob = 100;
-                    parseLinkResults(res.data);
                 }
-            } catch(err) {;
-                error = formatError(err);
+            } else if (res) {
+                progressJob = 100;
+                if (!$linkMapping || !$linkFileName || !$linkOptions.csv || !$linkSourceHeader) {
+                    const job = await getJobData($accessToken,$linkJob); //.filter(j => j.id === $linkJob)[0];
+                    console.log('Restore job metadata from server');
+                    $linkFileName = `${job.id}.csv`;
+                    $linkOptions.csv = {};
+                    $linkOptions.csv.sep = job.data && job.data.sep ;
+                    $linkOptions.csv.encoding = job.data.encoding;
+                    $linkOptions.csv.skipLines = Number(job.data.skipLines || "0");
+                    $linkOptions.csv.candidateNumber = Number(job.data.candidateNumber || "10");
+                    $linkOptions.csv.pruneScore = job.data.pruneScore;
+                    $linkOptions.csv.dateFormat = job.data.dateFormat;
+                    $linkOptions.csv.tmpfilePersistence = job.data && job.data.tmpfilePersistence;
+                    $linkOptions.type = "csv";
+                    $linkSourceHeader = validColumns.map(k => job.data && job.data[k]).filter(k => k);
+                    $linkMapping = { direct: {}, reverse: {} };
+                    validColumns.forEach(col => {
+                        if (job.data && job.data[col]) {
+                            $linkMapping.reverse[col] = job.data[col];
+                            $linkMapping.direct[job.data[col]] = col;
+                        }
+                    });
+                }
+                parseLinkResults(res.data);
             }
         }
     };
 
-    $: if (error) {$linkJob = 'failed'}
+    $: if (error) {
+        console.log('Error in job', error);
+        $linkJob = 'failed'
+    }
 
     $: if (progressDownload >= 100) { $linkWaiter = 'Préparation de la visualisation des résultats'; }
 
@@ -220,30 +253,37 @@
     }
 
     const formatError = (err) => {
-        let error;
+        let errorMessage;
+        console.log(err);
         if (err.response) {
-            error = `Erreur ${err.response.status}`;
+            console.log(err.response);
+            errorMessage = `Erreur ${err.response.status}`;
             if (err.response.data && err.response.data.msg) {
+                console.log(err.response.data.msg);
                 if (/column header mismatch/.test(err.response.data.msg)) {
-                    error += `<br>le nombre de colonnes n'est pas conforme à l'entête CSV`;
+                    errorMessage += `<br>le nombre de colonnes n'est pas conforme à l'entête CSV`;
+                } else if (err.response.status === 429 && /There is already \d+ running or waiting jobs/.test(err.response.data.msg)) {
+                    errorMessage += `<br>vous avez déja un traitement en cours - <a href="/jobs">consulter les traitements</a>`;
                 } else {
-                    error = error + '<br>' + JSON.stringify(err.response.data.msg);
+                    errorMessage = errorMessage + '<br>' + JSON.stringify(err.response.data.msg);
                 }
             } else {
                 if (err.response.status === 400) {
-                    error += `<br>problème dans le format des données, merci de nous contacter à ${mailTo}`
+                    errorMessage += `<br>problème dans le format des données, merci de nous contacter à ${mailTo}`
                 } else if (err.response.status === 502) {
-                    error += `<br>le serveur est indisponible, veuiller réessayer ultérieurement ou nous contacter à ${mailTo}`
+                    errorMessage += `<br>le serveur est indisponible, veuiller réessayer ultérieurement ou nous contacter à ${mailTo}`
                 } else if (err.response.status === 500) {
-                    error += `<br>le fichier a provoqué une erreur serveur, merci de nous contacter à ${mailTo}`
+                    errorMessage += `<br>le fichier a provoqué une erreur serveur, merci de nous contacter à ${mailTo}`
+                } else if (err.response.status === 429) {
+                    errorMessage += `<br>trop de requêtes, merci de recommencer ultérieurement`
                 } else {
-                    error += `<br>erreur inconnue, merci de nous contacter à ${mailTo}`
+                    errorMessage += `<br>erreur inconnue, merci de nous contacter à ${mailTo}`
                 }
             }
         } else {
-            error = `${err}`;
+            errorMessage = `${err}`;
         }
-        return error;
+        return errorMessage;
     }
 
     const parseLinkResults = (data) => {