From 56f35ee95ec990a4cd03d68408c8d7e5b0890e99 Mon Sep 17 00:00:00 2001 From: Kit Armstrong Date: Tue, 11 Aug 2020 08:26:39 -0700 Subject: [PATCH 1/2] NRPT-57: Allows multiple NRIS imports --- .../20200807181534-fixMineGuidDefault.js | 2 +- .../20200810162152-fixNrisSourceRef.js | 71 +++++++++++++++++++ api/src/controllers/post/inspection.js | 2 + api/src/integrations/nris-epd/datasource.js | 63 ++++++++++++++-- 4 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 api/migrations/20200810162152-fixNrisSourceRef.js diff --git a/api/migrations/20200807181534-fixMineGuidDefault.js b/api/migrations/20200807181534-fixMineGuidDefault.js index 23074464d..e5f370e38 100644 --- a/api/migrations/20200807181534-fixMineGuidDefault.js +++ b/api/migrations/20200807181534-fixMineGuidDefault.js @@ -14,7 +14,7 @@ exports.setup = function(options, seedLink) { seed = seedLink; }; -exports.up = function(db) { +exports.up = async function(db) { const mClient = await db.connection.connect(db.connectionString, { native_parser: true }) try { console.log('Starting fix for mineGUID.'); diff --git a/api/migrations/20200810162152-fixNrisSourceRef.js b/api/migrations/20200810162152-fixNrisSourceRef.js new file mode 100644 index 000000000..ddc49fde2 --- /dev/null +++ b/api/migrations/20200810162152-fixNrisSourceRef.js @@ -0,0 +1,71 @@ +'use strict'; + +const mongoose = require('mongoose'); +const ObjectId = mongoose.Types.ObjectId; + +let dbm; +let type; +let seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function(options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +exports.up = async function(db) { + const mClient = await db.connection.connect(db.connectionString, { native_parser: true }); + + try { + console.log('Starting fix for NRIS records...'); + + const nrpti = await mClient.collection('nrpti'); + + // Get all records that have been created from an NRIS import. + const records = await nrpti.find({ sourceSystemRef: 'nris-epd' }).toArray(); + + for (const record of records) { + if (!record.recordName){ + console.log(`Record ${record._id} is missing recordName`); + continue; + } + + // The record name contains the assessment ID at the end. + // It is in the format 'Inspection - {record name} - {assessment ID}' + const lastDash = record.recordName.lastIndexOf('-'); + + if (!lastDash){ + console.log(`Record ${record._id} has a different recordName format`); + continue; + } + + // There is always a space after the last dash. + const assessmentId = record.recordName.substring(lastDash + 2); + + if (!assessmentId){ + console.log(`Cannot find assessment ID for record ${record._id}`); + continue; + } + + await nrpti.update({ _id: new ObjectId(record._id) }, { $set: { _sourceRefNrisId: parseInt(assessmentId) }}); + } + + } catch (err) { + console.log('Error:', err); + } + + console.log('Done.'); + mClient.close() +}; + +exports.down = function(db) { + return null; +}; + +exports._meta = { + "version": 1 +}; diff --git a/api/src/controllers/post/inspection.js b/api/src/controllers/post/inspection.js index 52bba3128..dede11822 100644 --- a/api/src/controllers/post/inspection.js +++ b/api/src/controllers/post/inspection.js @@ -93,6 +93,8 @@ exports.createMaster = function(args, res, next, incomingObj, flavourIds) { (inspection._sourceRefOgcInspectionId = incomingObj._sourceRefOgcInspectionId); incomingObj._sourceRefOgcDeficiencyId && (inspection._sourceRefOgcDeficiencyId = incomingObj._sourceRefOgcDeficiencyId); + incomingObj._sourceRefNrisId && + (inspection._sourceRefNrisId = incomingObj._sourceRefNrisId); // set permissions inspection.read = ROLES.ADMIN_ROLES; diff --git a/api/src/integrations/nris-epd/datasource.js b/api/src/integrations/nris-epd/datasource.js index 45070701b..940ccaa35 100644 --- a/api/src/integrations/nris-epd/datasource.js +++ b/api/src/integrations/nris-epd/datasource.js @@ -143,7 +143,16 @@ class NrisDataSource { moment().diff(moment(records[i].completionDate), 'days') > 7 ) { const newRecord = await this.transformRecord(records[i]); - await this.createRecord(newRecord); + + const existingRecord = await this.findExistingRecord(newRecord); + + if (existingRecord) { + await this.updateRecord(newRecord, existingRecord); + } else { + await this.createRecordAttachments(records[i], newRecord); + await this.createRecord(newRecord); + } + // Assuming we didn't get thrown an error, update the items successfully processed. processingObject.itemsProcessed++; } else { @@ -176,7 +185,7 @@ class NrisDataSource { newRecord.recordName = `Inspection - ${record.requirementSource} - ${record.assessmentId}`; newRecord.legislationDescription = 'Inspection to verify compliance with regulatory requirement.'; newRecord.recordType = 'Inspection'; - newRecord._sourceRefNrisId = record.assessmentId; + newRecord._sourceRefNrisId = null newRecord.dateIssued = record.assessmentDate; newRecord.issuingAgency = this.stringTransformEPOtoEPD(record.resourceAgency); newRecord.author = 'Environmental Protection Division'; @@ -233,6 +242,12 @@ class NrisDataSource { newRecord.outcomeDescription += ' - ' + record.inspection.inspctResponse; } + defaultLog.info('Processed:', record.assessmentId); + return newRecord; + } + + // Gets the files and saves them for a record's attachment. + async createRecordAttachments(record, newRecord) { for (let i = 0; i < record.attachment.length; i++) { if (record.attachment[i].fileType === 'Final Report') { const tempFileData = await this.getFileFromNRIS(record.assessmentId, record.attachment[i].attachmentId); @@ -244,9 +259,6 @@ class NrisDataSource { } } } - - defaultLog.info('Processed:', record.assessmentId); - return newRecord; } // Grabs a file from NRIS datasource @@ -350,6 +362,47 @@ class NrisDataSource { defaultLog.error(`Failed to create Inspection record: ${error.message}`); } } + + // Checks to see if a record already exists. + async findExistingRecord(transformedRecord) { + const masterRecordModel = mongoose.model(RECORD_TYPE.Inspection._schemaName); + + return await masterRecordModel + .findOne({ + _schemaName: RECORD_TYPE.Inspection._schemaName, + _sourceRefNrisId: transformedRecord._sourceRefNrisId + }) + .populate('_flavourRecords'); + } + + // Updates an existing record with new data pulled from the API. + async updateRecord(newRecord, existingRecord) { + if (!newRecord) { + throw Error('updateRecord - required newRecord must be non-null.'); + } + + if (!existingRecord) { + throw Error('updateRecord - required existingRecord must be non-null.'); + } + + try { + // build update Obj, which needs to include the flavour record ids + const updateObj = { ...newRecord, _id: existingRecord._id }; + existingRecord._flavourRecords.forEach(flavourRecord => { + updateObj[flavourRecord._schemaName] = { _id: flavourRecord._id, addRole: 'public' }; + }); + + return await RecordController.processPutRequest( + { swagger: { params: { auth_payload: this.auth_payload } } }, + null, + null, + RECORD_TYPE.Inspection.recordControllerName, + [updateObj] + ); + } catch (error) { + defaultLog.error(`Failed to save ${RECORD_TYPE.Inspection._schemaName} record: ${error.message}`); + } + } } module.exports = NrisDataSource; From 2229ab5854e411e13a2c3cc400e0222a01f202a4 Mon Sep 17 00:00:00 2001 From: Kit Armstrong Date: Tue, 11 Aug 2020 09:57:09 -0700 Subject: [PATCH 2/2] NRPT-327: Fixes PermitBCMI Documents --- api/src/models/bcmi/permit-bcmi.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/api/src/models/bcmi/permit-bcmi.js b/api/src/models/bcmi/permit-bcmi.js index 71f7e229e..8166ab9e7 100644 --- a/api/src/models/bcmi/permit-bcmi.js +++ b/api/src/models/bcmi/permit-bcmi.js @@ -27,12 +27,8 @@ module.exports = require('../../utils/model-schema-generator')( dateIssued: { type: Date, default: null }, authorizedEndDate: { type: Date, default: null }, description: { type: String, default: '' }, - // Document ref to meta record and physical document on the object store - amendmentDocument: { - _sourceRefId: { type: String, default: null }, - documentName: { type: String, defualt: '' }, - documentId: { type: 'ObjectId', default: null } - }, + documents: [{ type: 'ObjectId', default: [], index: true }], + // final Record boilerplate dateAdded: { type: Date, default: Date.now() }, dateUpdated: { type: Date, default: null },