Generate return logs from requirements (#1244)

https://eaflood.atlassian.net/browse/WATER-4543

As part of the work to replace the legacy system NALD we need to generate return logs. This PR creates a job that when run twice a year will create the return logs for the appropriate cycle, either summer or winter/all year.

app/controllers/jobs.controller.js

@@ -9,6 +9,10 @@ const ExportService = require('../services/jobs/export/export.service.js')
 const ProcessLicenceUpdates = require('../services/jobs/licence-updates/process-licence-updates.js')
 const ProcessSessionStorageCleanupService = require('../services/jobs/session-cleanup/process-session-storage-cleanup.service.js')
 const ProcessTimeLimitedLicencesService = require('../services/jobs/time-limited/process-time-limited-licences.service.js')
+const ProcessReturnLogsService = require('../services/jobs/return-logs/process-return-logs.service.js')
+
+const redirectStatusCode = 204
+const notFoundStatusCode = 404
 
 /**
  * Triggers export of all relevant tables to CSV and then uploads them to S3
@@ -20,30 +24,49 @@ const ProcessTimeLimitedLicencesService = require('../services/jobs/time-limited
 async function exportDb (_request, h) {
   ExportService.go()
 
-  return h.response().code(204)
+  return h.response().code(redirectStatusCode)
 }
 
 async function licenceUpdates (_request, h) {
   ProcessLicenceUpdates.go()
 
-  return h.response().code(204)
+  return h.response().code(redirectStatusCode)
 }
 
 async function sessionCleanup (_request, h) {
   ProcessSessionStorageCleanupService.go()
 
-  return h.response().code(204)
+  return h.response().code(redirectStatusCode)
 }
 
 async function timeLimited (_request, h) {
   ProcessTimeLimitedLicencesService.go()
 
-  return h.response().code(204)
+  return h.response().code(redirectStatusCode)
+}
+
+async function returnLogs (request, h) {
+  const { cycle } = request.params
+
+  if (!['summer', 'all-year'].includes(cycle)) {
+    return h.response().code(notFoundStatusCode)
+  }
+
+  let licenceReference
+
+  if (h.request.payload !== null && h.request.payload.licenceReference) {
+    licenceReference = h.request.payload.licenceReference
+  }
+
+  ProcessReturnLogsService.go(cycle, licenceReference)
+
+  return h.response().code(redirectStatusCode)
 }
 
 module.exports = {
   exportDb,
   licenceUpdates,
+  returnLogs,
   sessionCleanup,
   timeLimited
 }

app/routes/jobs.routes.js

@@ -58,6 +58,20 @@ const routes = [
         crumb: false
       }
     }
+  },
+  {
+    method: 'POST',
+    path: '/jobs/return-logs/{cycle}',
+    options: {
+      handler: JobsController.returnLogs,
+      app: {
+        plainOutput: true
+      },
+      auth: false,
+      plugins: {
+        crumb: false
+      }
+    }
   }
 ]
 

app/services/jobs/return-logs/fetch-return-logs.service.js

@@ -0,0 +1,243 @@
+'use strict'
+
+/**
+ * Fetches data needed for the generating return logs
+ * @module FetchLicenceWithoutReturnsService
+ */
+
+const ReturnLogModel = require('../../../models/return-log.model.js')
+const ReturnRequirementModel = require('../../../models/return-requirement.model.js')
+const ReturnVersionModel = require('../../../models/return-version.model.js')
+
+const { db } = require('../../../../db/db.js')
+
+const allYearDueDateDay = 28
+const allYearDueDateMonth = 3
+const allYearEndDay = 31
+const allYearEndMonth = 2
+const allYearStartDay = 1
+const allYearStartMonth = 3
+
+const summerDueDateDay = 28
+const summerDueDateMonth = 10
+const summerEndDay = 31
+const summerEndMonth = 9
+const summerStartDay = 1
+const summerStartMonth = 10
+
+const endOfSummerCycle = new Date(new Date().getFullYear() + 1, summerEndMonth, summerEndDay)
+const endOfWinterAndAllYearCycle = new Date(new Date().getFullYear() + 1, allYearEndMonth, allYearEndDay)
+
+/**
+ * Fetch all return requirements that need return logs created.
+ *
+ * @returns {Promise<Array>} the list of return requirement ids
+ */
+async function go (isSummer, licenceReference) {
+  const requirementsForReturns = await _fetchReturnRequirements(isSummer, licenceReference)
+  const data = await _generateReturnLogPayload(isSummer, requirementsForReturns)
+
+  return data
+}
+
+async function _createMetaData (isSummer, endDate, requirements) {
+  return {
+    description: requirements.siteDescription,
+    isCurrent: requirements.returnVersion.reason !== 'succession-or-transfer-of-licence',
+    isFinal: _isFinal(endDate, isSummer),
+    isSummer,
+    isTwoPartTariff: requirements.twoPartTariff,
+    isUpload: requirements.upload,
+    nald: {
+      regionCode: requirements.returnVersion.licence.region.naldRegionId,
+      areaCode: requirements.returnVersion.licence.areacode,
+      formatId: requirements.legacyId,
+      periodStartDay: requirements.abstractionPeriodStartDay,
+      periodStartMonth: requirements.abstractionPeriodStartMonth,
+      periodEndDay: requirements.abstractionPeriodEndDay,
+      periodEndMonth: requirements.abstractionPeriodEndMonth
+    },
+    points: requirements.returnRequirementPoints,
+    purposes: requirements.returnRequirementPurposes,
+    version: 1
+  }
+}
+
+function _createReturnLogId (requirements, startDate, endDate) {
+  const regionCode = requirements.returnVersion.licence.region.naldRegionId
+  const licenceReference = requirements.returnVersion.licence.licenceRef
+  const legacyId = requirements.legacyId
+
+  return `v1:${regionCode}:${licenceReference}:${legacyId}:${startDate}:${endDate}`
+}
+
+async function _fetchExternalIds (cycleStartDate) {
+  const externalIds = await ReturnLogModel.query()
+    .select(['licenceRef',
+      db.raw("concat(ret.metadata->'nald'->>'regionCode', ':', ret.return_requirement) as externalid")
+    ])
+    .from('returns.returns as ret')
+    .where('startDate', '>=', cycleStartDate)
+
+  const externalIdsArray = externalIds.map((item) => {
+    return item.externalid
+  })
+
+  return externalIdsArray
+}
+
+async function _fetchReturnRequirements (isSummer, licenceReference) {
+  const cycleStartDate = _getCycleStartDate(isSummer)
+  const externalIds = await _fetchExternalIds(cycleStartDate)
+
+  const results = await ReturnRequirementModel.query()
+    .whereNotIn('returnRequirements.externalId', externalIds)
+    .whereExists(_whereExistsClause(licenceReference, cycleStartDate))
+    .where('returnRequirements.summer', isSummer)
+    .withGraphFetched('returnVersion')
+    .modifyGraph('returnVersion', (builder) => {
+      builder.select(['endDate',
+        'id',
+        'startDate',
+        'reason'])
+    })
+    .withGraphFetched('returnVersion.licence')
+    .modifyGraph('returnVersion.licence', (builder) => {
+      builder.select(['expiredDate',
+        'id',
+        'lapsedDate',
+        'licenceRef',
+        'revokedDate',
+        db.raw('regions->>\'historicalAreaCode\' as areacode')])
+    })
+    .withGraphFetched('returnVersion.licence.region')
+    .modifyGraph('returnVersion.licence.region', (builder) => {
+      builder.select(['id', 'naldRegionId'])
+    })
+    .withGraphFetched('returnRequirementPoints')
+    .withGraphFetched('returnRequirementPurposes')
+
+  return results
+}
+
+function _formatDate (date) {
+  return date.toISOString().split('T')[0]
+}
+
+async function _generateReturnLogPayload (isSummer, requirementsForReturns) {
+  const returnLogs = requirementsForReturns.map(async (requirements) => {
+    const startDate = _getCycleStartDate(isSummer)
+    const endDate = _getCycleEndDate(isSummer, requirements.returnVersion)
+    const id = _createReturnLogId(requirements, startDate, endDate)
+    const metadata = await _createMetaData(isSummer, endDate, requirements)
+
+    return {
+      createdAt: new Date(),
+      dueDate: _getCycleDueDate(isSummer),
+      endDate,
+      id,
+      licenceRef: requirements.returnVersion.licence.licenceRef,
+      metadata,
+      returnsFrequency: requirements.reportingFrequency,
+      startDate,
+      status: 'due',
+      source: 'WRLS',
+      returnReference: requirements.legacyId.toString()
+    }
+  })
+
+  const results = await Promise.all(returnLogs)
+
+  return results
+}
+
+function _getCycleDueDate (isSummer) {
+  return isSummer
+    ? _formatDate(new Date(new Date().getFullYear() + 1, summerDueDateMonth, summerDueDateDay))
+    : _formatDate(new Date(new Date().getFullYear() + 1, allYearDueDateMonth, allYearDueDateDay))
+}
+
+function _getCycleEndDate (isSummer, returnVersion) {
+  const dates = [returnVersion.licence.expiredDate,
+    returnVersion.licence.lapsedDate,
+    returnVersion.licence.revokedDate,
+    returnVersion.endDate]
+    .filter((date) => { return date !== null })
+    .map((date) => { return new Date(date) })
+
+  if (dates.length === 0) {
+    return isSummer
+      ? _formatDate(endOfSummerCycle)
+      : _formatDate(endOfWinterAndAllYearCycle)
+  }
+
+  dates.map((date) => { return date.getTime() })
+  const earliestEndDate = new Date(Math.min(...dates))
+
+  if (isSummer) {
+    if (earliestEndDate < endOfSummerCycle) {
+      return _formatDate(earliestEndDate)
+    }
+
+    return _formatDate(endOfSummerCycle)
+  }
+
+  if (earliestEndDate < endOfWinterAndAllYearCycle) {
+    return _formatDate(earliestEndDate)
+  }
+
+  return _formatDate(endOfWinterAndAllYearCycle)
+}
+
+function _getCycleStartDate (isSummer) {
+  return isSummer
+    ? _formatDate(new Date(new Date().getFullYear(), summerStartMonth, summerStartDay))
+    : _formatDate(new Date(new Date().getFullYear(), allYearStartMonth, allYearStartDay))
+}
+
+function _isFinal (endDateString, isSummer) {
+  const endDate = new Date(endDateString)
+
+  return ((isSummer && endDate < endOfSummerCycle) || (!isSummer && endDate < endOfWinterAndAllYearCycle))
+}
+
+function _whereExistsClause (licenceReference, cycleStartDate) {
+  const query = ReturnVersionModel.query().select(1)
+
+  query.select(1)
+    .innerJoinRelated('licence')
+    .where('returnVersions.startDate', '<=', cycleStartDate)
+    .where('returnVersions.status', 'current')
+    .where((builder) => {
+      builder
+        .whereNull('returnVersions.endDate')
+        .orWhere('returnVersions.endDate', '>=', cycleStartDate)
+    })
+    .where((builder) => {
+      builder
+        .whereNull('licence.expiredDate')
+        .orWhere('licence.expiredDate', '>=', cycleStartDate)
+    })
+    .where((builder) => {
+      builder
+        .whereNull('licence.lapsedDate')
+        .orWhere('licence.lapsedDate', '>=', cycleStartDate)
+    })
+    .where((builder) => {
+      builder
+        .whereNull('licence.revokedDate')
+        .orWhere('licence.revokedDate', '>=', cycleStartDate)
+    })
+
+  query.whereColumn('returnVersions.id', 'returnRequirements.returnVersionId')
+
+  if (licenceReference) {
+    query.where('licence.licenceRef', licenceReference)
+  }
+
+  return query
+}
+
+module.exports = {
+  go
+}

app/services/jobs/return-logs/process-return-logs.service.js

@@ -0,0 +1,58 @@
+'use strict'
+
+/**
+ * Process the return logs for the next cycle
+ * @module ProcessReturnLogsService
+ */
+
+const { calculateAndLogTimeTaken, currentTimeInNanoseconds } = require('../../../lib/general.lib.js')
+const FetchReturnLogsService = require('./fetch-return-logs.service.js')
+const ReturnLogModel = require('../../../models/return-log.model.js')
+
+/**
+ * Creates the return logs for the next cycle
+ * The return requirement is the information held against the licence that defines how and when an abstractor needs to
+ * submit their returns.
+ *
+ * The return log is the 'header' record generated each return cycle from the requirement that an abstractor submits
+ * their returns against.
+ *
+ * When users make changes to return requirements, the service will determine if any new return logs need to be
+ * created depending on the current cycle.
+ *
+ * But if no changes are ever made to a licence's return requirements, and this job didn't exist, no new return logs
+ * would be created.
+ *
+ * So, this job will run twice yearly: once for each cycle. The job determines which return requirements need a return
+ * log generated for the selected cycle and then creates them.
+ *
+ * > Because the job creates _all_ return logs in a cycle, it makes it difficult to test what it is generating is
+ * > correct. So, to support testing and validation, we can pass a licence ref in the job request to limit the creation
+ * > to just a single licence.
+ * @param {string} cycle - the return cycle to create logs for (summer or all-year)
+ * @param {string} [licenceReference] - An optional argument to limit return log creation to just the specific licence
+ */
+async function go (cycle, licenceReference = null) {
+  try {
+    const startTime = currentTimeInNanoseconds()
+    const isSummer = cycle === 'summer'
+    const returnLogs = await FetchReturnLogsService.go(isSummer, licenceReference)
+
+    await _createReturnLogs(returnLogs)
+
+    calculateAndLogTimeTaken(startTime, 'Create return logs job complete', { cycle, licenceReference })
+  } catch (error) {
+    global.GlobalNotifier.omfg('Create return logs job failed', { cycle, error })
+  }
+}
+
+async function _createReturnLogs (returnLogs) {
+  for (const returnLog of returnLogs) {
+    await ReturnLogModel.query()
+      .insert(returnLog)
+  }
+}
+
+module.exports = {
+  go
+}