diff --git a/app/services/health/fetch-import-jobs.service.js b/app/services/health/fetch-import-jobs.service.js new file mode 100644 index 0000000000..bdf566bedf --- /dev/null +++ b/app/services/health/fetch-import-jobs.service.js @@ -0,0 +1,90 @@ +'use strict' + +/** + * Fetches details of the pg-boss jobs run by the Import service in the last 24hrs + * @module FetchImportJobs + */ + +const { db } = require('../../../db/db.js') + +/** + * Returns data required to populate the Import tasks of our `/health/info` page. +*/ +async function go () { + const PGBOSS_JOBS_ARRAY = [ + 'import.bill-runs', + 'import.charge-versions', + 'import.charging-data', + 'import.tracker', + 'licence-import.delete-removed-documents', + 'licence-import.import-company', + 'licence-import.import-licence', + 'licence-import.import-purpose-condition-types', + 'licence-import.queue-companies', + 'licence-import.queue-licences', + 'nald-import.delete-removed-documents', + 'nald-import.import-licence', + 'nald-import.queue-licences', + 'nald-import.s3-download' + ] + const currentDateMinusOneDay = _subtractDaysFromCurrentDate(1) + + return db + .select('name') + .count({ completedCount: db.raw("CASE WHEN state = 'completed' THEN 1 END") }) + .count({ failedCount: db.raw("CASE WHEN state = 'failed' THEN 1 END") }) + .count({ activeCount: db.raw("CASE WHEN state IN ('active', 'created') THEN 1 END") }) + .max('completedon as maxCompletedonDate') + .from( + (db + .select( + 'name', + 'state', + 'completedon' + ) + .from('water_import.job') + .whereIn('state', ['failed', 'completed', 'active', 'created']) + .whereIn('name', PGBOSS_JOBS_ARRAY) + .where((builder) => + builder + .where('createdon', '>', currentDateMinusOneDay) + .orWhere('completedon', '>', currentDateMinusOneDay) + ) + .unionAll( + db + .select( + 'name', + 'state', + 'completedon' + ) + .from('water_import.archive') + .whereIn('state', ['failed', 'completed', 'active', 'created']) + .whereIn('name', PGBOSS_JOBS_ARRAY) + .where((builder) => + builder + .where('createdon', '>', currentDateMinusOneDay) + .orWhere('completedon', '>', currentDateMinusOneDay) + ) + )) + .as('jobs') + ) + .groupBy('name') +} + +/** + * Calculates the current date minus the number of days passed to it + * + * @param {Number} days The number of days to be deducted from the currect date + * + * @returns {Date} The current date minus the number days passed to the function + */ +function _subtractDaysFromCurrentDate (days) { + const date = new Date() + date.setDate(date.getDate() - days) + + return date +} + +module.exports = { + go +} diff --git a/app/services/health/info.service.js b/app/services/health/info.service.js index bbae91b4fb..0e6fb32457 100644 --- a/app/services/health/info.service.js +++ b/app/services/health/info.service.js @@ -12,6 +12,8 @@ const exec = util.promisify(ChildProcess.exec) const redis = require('@redis/client') const ChargingModuleRequestLib = require('../../lib/charging-module-request.lib.js') +const FetchImportJobs = require('./fetch-import-jobs.service.js') +const { formatLongDateTime } = require('../../presenters/base.presenter.js') const RequestLib = require('../../lib/request.lib.js') const LegacyRequestLib = require('../../lib/legacy-request.lib.js') @@ -26,58 +28,18 @@ const servicesConfig = require('../../../config/services.config.js') * array per row, where each row array contains multiple `{ text: '...' }` elements, one for each cell in the row. */ async function go () { - const virusScannerData = await _getVirusScannerData() - const redisConnectivityData = await _getRedisConnectivityData() - const addressFacadeData = await _getAddressFacadeData() - const chargingModuleData = await _getChargingModuleData() const appData = await _getAppData() + const chargingModuleData = await _getChargingModuleData() + const redisConnectivityData = await _getRedisConnectivityData() + const virusScannerData = await _getVirusScannerData() return { - virusScannerData, - redisConnectivityData, addressFacadeData, + appData, chargingModuleData, - appData - } -} - -/** - * Receives an array and returns it in the format required by the nunjucks template in the view. - */ -function _mapArrayToTextCells (rows) { - return rows.map(row => { - return row.map(cell => { - return { text: cell } - }) - }) -} - -async function _getVirusScannerData () { - try { - const { stdout, stderr } = await exec('clamdscan --version') - return stderr ? `ERROR: ${stderr}` : stdout - } catch (error) { - return `ERROR: ${error.message}` - } -} - -async function _getRedisConnectivityData () { - try { - const client = redis.createClient({ - socket: { - host: redisConfig.host, - port: redisConfig.port - }, - password: redisConfig.password - }) - - await client.connect() - await client.disconnect() - - return 'Up and running' - } catch (error) { - return 'Error connecting to Redis' + redisConnectivityData, + virusScannerData } } @@ -92,43 +54,20 @@ async function _getAddressFacadeData () { return _parseFailedRequestResult(result) } -async function _getChargingModuleData () { - const result = await ChargingModuleRequestLib.get('status') - - if (result.succeeded) { - return result.response.info.dockerTag - } - - return _parseFailedRequestResult(result) -} - -function _getImportJobsData () { - return _mapArrayToTextCells([ - [ - 'Cell 1.1', - 'Cell 1.2' - ], - [ - 'Cell 2.1', - 'Cell 2.2' - ] - ]) -} - async function _getAppData () { const healthInfoPath = 'health/info' const services = [ - { name: 'Service - foreground', serviceName: 'water' }, - { name: 'Service - background', serviceName: 'background' }, - { name: 'Reporting', serviceName: 'reporting' }, { name: 'Import', serviceName: 'import' }, - { name: 'Tactical CRM', serviceName: 'crm' }, { name: 'External UI', serviceName: 'external' }, { name: 'Internal UI', serviceName: 'internal' }, + { name: 'Service - foreground', serviceName: 'water' }, + { name: 'Service - background', serviceName: 'background' }, + { name: 'Returns', serviceName: 'returns' }, + { name: 'Tactical CRM', serviceName: 'crm' }, { name: 'Tactical IDM', serviceName: 'idm' }, - { name: 'Permit repository', serviceName: 'permits' }, - { name: 'Returns', serviceName: 'returns' } + { name: 'Reporting', serviceName: 'reporting' }, + { name: 'Permit repository', serviceName: 'permits' } ] for (const service of services) { @@ -137,7 +76,7 @@ async function _getAppData () { if (result.succeeded) { service.version = result.response.body.version service.commit = result.response.body.commit - service.jobs = service.name === 'Import' ? _getImportJobsData() : [] + service.jobs = service.name === 'Import' ? await _getImportJobsData() : [] } else { service.version = _parseFailedRequestResult(result) service.commit = '' @@ -147,6 +86,67 @@ async function _getAppData () { return services } +async function _getChargingModuleData () { + const result = await ChargingModuleRequestLib.get('status') + + if (result.succeeded) { + return result.response.info.dockerTag + } + + return _parseFailedRequestResult(result) +} + +async function _getImportJobsData () { + try { + const importJobs = await FetchImportJobs.go() + + return _mapArrayToTextCells(importJobs) + } catch (error) { + return [] + } +} + +async function _getRedisConnectivityData () { + try { + const client = redis.createClient({ + socket: { + host: redisConfig.host, + port: redisConfig.port + }, + password: redisConfig.password + }) + + await client.connect() + await client.disconnect() + + return 'Up and running' + } catch (error) { + // `error` value not returned for security reasons as it gives out the IP address and port of Redis + return 'Error connecting to Redis' + } +} + +async function _getVirusScannerData () { + try { + const { stdout, stderr } = await exec('clamdscan --version') + return stderr ? `ERROR: ${stderr}` : stdout + } catch (error) { + return `ERROR: ${error.message}` + } +} + +function _mapArrayToTextCells (rows) { + return rows.map(row => { + return [ + ...[{ text: row.name }], + ...[{ text: row.completedCount }], + ...[{ text: row.failedCount }], + ...[{ text: row.activeCount }], + ...[{ text: row.maxCompletedonDate ? formatLongDateTime(row.maxCompletedonDate) : '-' }] + ] + }) +} + function _parseFailedRequestResult (result) { if (result.response.statusCode) { return `ERROR: ${result.response.statusCode} - ${result.response.body}` diff --git a/app/views/info.njk b/app/views/info.njk index 450a339f64..31b4ed62a6 100644 --- a/app/views/info.njk +++ b/app/views/info.njk @@ -56,10 +56,19 @@ firstCellIsHeader: false, head: [ { - text: "Job name" + text: "Import task name (data for the last 24 hrs)" }, { - text: "Last updated" + text: "Completed" + }, + { + text: "Failed" + }, + { + text: "Active" + }, + { + text: "Last job completed" } ], rows: app.jobs diff --git a/test/services/health/info.service.test.js b/test/services/health/info.service.test.js index ea0954f88f..0236525c77 100644 --- a/test/services/health/info.service.test.js +++ b/test/services/health/info.service.test.js @@ -14,6 +14,7 @@ const servicesConfig = require('../../../config/services.config.js') // Things we need to stub const ChargingModuleRequestLib = require('../../../app/lib/charging-module-request.lib.js') +const FetchImportJobs = require('../../../app/services/health/fetch-import-jobs.service.js') const LegacyRequestLib = require('../../../app/lib/legacy-request.lib.js') const redis = require('@redis/client') const RequestLib = require('../../../app/lib/request.lib.js') @@ -38,13 +39,33 @@ describe('Info service', () => { }, app: { succeeded: true, response: { statusCode: 200, body: { version: '9.0.99', commit: '99d0e8c' } } } } + + const goodFetchImportJobsResults = [ + { + name: 'import.charging-data', + completedCount: 1, + failedCount: 0, + activeCount: 0, + maxCompletedonDate: new Date('2023-09-04T14:00:10.758Z') + }, + { + name: 'licence-import.import-company', + completedCount: 52963, + failedCount: 1, + activeCount: 123, + maxCompletedonDate: new Date('2023-09-04T10:43:44.503Z') + } + ] + let chargingModuleRequestLibStub + let fetchImportJobsStub let legacyRequestLibStub let requestLibStub let redisStub beforeEach(() => { chargingModuleRequestLibStub = Sinon.stub(ChargingModuleRequestLib, 'get') + fetchImportJobsStub = Sinon.stub(FetchImportJobs, 'go') legacyRequestLibStub = Sinon.stub(LegacyRequestLib, 'get') requestLibStub = Sinon.stub(RequestLib, 'get') redisStub = Sinon.stub(redis, 'createClient') @@ -64,8 +85,6 @@ describe('Info service', () => { chargingModuleRequestLibStub .withArgs('status') .resolves(goodRequestResults.chargingModule) - - // redisStub.returns({ connect: Sinon.fake().resolves(), disconnect: Sinon.fake().resolves() }) }) afterEach(() => { @@ -74,6 +93,9 @@ describe('Info service', () => { describe('when all the services are running', () => { beforeEach(async () => { + fetchImportJobsStub.resolves(goodFetchImportJobsResults) + redisStub.returns({ connect: Sinon.stub().resolves(), disconnect: Sinon.stub().resolves() }) + // In this scenario everything is hunky-dory so we return 2xx responses from these services requestLibStub .withArgs(`${servicesConfig.addressFacade.url}/address-service/hola`) @@ -95,8 +117,6 @@ describe('Info service', () => { }) const utilStub = { promisify: Sinon.stub().callsFake(() => execStub) } InfoService = Proxyquire('../../../app/services/health/info.service', { util: utilStub }) - - redisStub.returns({ connect: Sinon.stub().resolves(), disconnect: Sinon.stub().resolves() }) }) it('returns details on each', async () => { @@ -107,14 +127,37 @@ describe('Info service', () => { ]) expect(result.appData).to.have.length(10) + expect(result.appData[0].name).to.equal('Import') + expect(result.appData[0].serviceName).to.equal('import') + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].commit).to.equal('99d0e8c') + expect(result.appData[0].jobs).to.equal([ + [ + { text: 'import.charging-data' }, + { text: 1 }, + { text: 0 }, + { text: 0 }, + { text: '4 September 2023 at 14:00:10' } + ], + [ + { text: 'licence-import.import-company' }, + { text: 52963 }, + { text: 1 }, + { text: 123 }, + { text: '4 September 2023 at 10:43:44' } + ] + ]) + expect(result.appData[1].jobs).to.equal([]) - expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') expect(result.redisConnectivityData).to.equal('Up and running') + expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') }) }) describe('when Redis', () => { beforeEach(async () => { + fetchImportJobsStub.resolves(goodFetchImportJobsResults) + // In these scenarios everything is hunky-dory so we return 2xx responses from these services requestLibStub .withArgs(`${servicesConfig.addressFacade.url}/address-service/hola`) @@ -147,22 +190,25 @@ describe('Info service', () => { 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' ]) expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(2) - expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') expect(result.redisConnectivityData).to.equal('Error connecting to Redis') + expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') }) }) }) describe('when ClamAV', () => { beforeEach(async () => { + fetchImportJobsStub.resolves(goodFetchImportJobsResults) + redisStub.returns({ connect: Sinon.stub().resolves(), disconnect: Sinon.stub().resolves() }) + // In these scenarios everything is hunky-dory so we return 2xx responses from these services requestLibStub .withArgs(`${servicesConfig.addressFacade.url}/address-service/hola`) .resolves(goodRequestResults.addressFacade) legacyRequestLibStub.withArgs('water', 'health/info', false).resolves(goodRequestResults.app) - - redisStub.returns({ connect: Sinon.stub().resolves(), disconnect: Sinon.stub().resolves() }) }) describe('is not running', () => { @@ -187,9 +233,11 @@ describe('Info service', () => { 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' ]) expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(2) - expect(result.virusScannerData).to.startWith('ERROR:') expect(result.redisConnectivityData).to.equal('Up and running') + expect(result.virusScannerData).to.startWith('ERROR:') }) }) @@ -212,6 +260,8 @@ describe('Info service', () => { 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' ]) expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(2) expect(result.virusScannerData).to.startWith('ERROR:') expect(result.redisConnectivityData).to.equal('Up and running') @@ -219,8 +269,72 @@ describe('Info service', () => { }) }) + describe('when FetchImportJobs service', () => { + beforeEach(async () => { + redisStub.returns({ connect: Sinon.stub().resolves(), disconnect: Sinon.stub().resolves() }) + + // In this scenario everything is hunky-dory so we return 2xx responses from these services + requestLibStub + .withArgs(`${servicesConfig.addressFacade.url}/address-service/hola`) + .resolves(goodRequestResults.addressFacade) + legacyRequestLibStub.withArgs('water', 'health/info', false).resolves(goodRequestResults.app) + + const execStub = Sinon + .stub() + .withArgs('clamdscan --version') + .resolves({ + stdout: 'ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n', + stderror: null + }) + const utilStub = { promisify: Sinon.stub().callsFake(() => execStub) } + InfoService = Proxyquire('../../../app/services/health/info.service', { util: utilStub }) + }) + + describe('returns no results', () => { + beforeEach(async () => { + fetchImportJobsStub.resolves([]) + }) + + it('jobs are empty and still returns a result for the other services', async () => { + const result = await InfoService.go() + + expect(result).to.include([ + 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' + ]) + expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(0) + + expect(result.redisConnectivityData).to.equal('Up and running') + expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') + }) + }) + + describe('throws an exception', () => { + beforeEach(async () => { + fetchImportJobsStub.throwsException(new Error('FetchImportJobs went boom')) + }) + + it('handles the error and still returns a result for the other services', async () => { + const result = await InfoService.go() + + expect(result).to.include([ + 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' + ]) + expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(0) + + expect(result.redisConnectivityData).to.equal('Up and running') + expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') + }) + }) + }) + describe('when a service we check via http request', () => { beforeEach(async () => { + fetchImportJobsStub.resolves(goodFetchImportJobsResults) + // In these scenarios everything is hunky-dory with clamav and redis. So, we go back to our original stubbing const execStub = Sinon .stub() @@ -252,9 +366,10 @@ describe('Info service', () => { 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' ]) expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(2) expect(result.addressFacadeData).to.startWith('ERROR:') - expect(result.appData[0].version).to.startWith('ERROR:') expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') expect(result.redisConnectivityData).to.equal('Up and running') @@ -278,9 +393,10 @@ describe('Info service', () => { 'virusScannerData', 'redisConnectivityData', 'addressFacadeData', 'chargingModuleData', 'appData' ]) expect(result.appData).to.have.length(10) + expect(result.appData[0].version).to.equal('9.0.99') + expect(result.appData[0].jobs).to.have.length(2) expect(result.addressFacadeData).to.startWith('ERROR:') - expect(result.appData[0].version).to.startWith('ERROR:') expect(result.virusScannerData).to.equal('ClamAV 9.99.9/26685/Mon Oct 10 08:00:01 2022\n') expect(result.redisConnectivityData).to.equal('Up and running')