-
Notifications
You must be signed in to change notification settings - Fork 106
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #238 from cumulus-nasa/CUMULUS-371
end-to-end tests in circleci [CUMULUS-371]
- Loading branch information
Showing
13 changed files
with
612 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| /** | ||
| * Includes helper functions for replicating Step Function Workflows | ||
| * locally | ||
| */ | ||
| 'use strict'; | ||
|
|
||
| const path = require('path'); | ||
| const fs = require('fs-extra'); | ||
| const clone = require('lodash.clonedeep'); | ||
| const { randomString } = require('@cumulus/common/test-utils'); | ||
| const { template } = require('@cumulus/deployment/lib/message'); | ||
| const { fetchMessageAdapter } = require('@cumulus/deployment/lib/adapter'); | ||
|
|
||
| /** | ||
| * Download cumulus message adapter (CMA) and unzip it | ||
| * | ||
| * @param {string} version - cumulus message adapter version number (optional) | ||
| * @returns {Promise.<Object>} an object with path to the zip and extracted CMA | ||
| */ | ||
| async function downloadCMA(version) { | ||
| // download and unzip the message adapter | ||
| const gitPath = 'cumulus-nasa/cumulus-message-adapter'; | ||
| const filename = 'cumulus-message-adapter.zip'; | ||
| const src = path.join(process.cwd(), 'tests', `${randomString()}.zip`); | ||
| const dest = path.join(process.cwd(), 'tests', randomString()); | ||
| await fetchMessageAdapter(version, gitPath, filename, src, dest); | ||
| return { | ||
| src, | ||
| dest | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Copy cumulus message adapter python folder to each task | ||
| * in the workflow | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {string} src - the path to the cumulus message adapter folder | ||
| * @param {string} cmaFolder - the name of the folder where CMA is copied to | ||
| * @returns {Promise.<Array>} an array of undefined values | ||
| */ | ||
| function copyCMAToTasks(workflow, src, cmaFolder) { | ||
| return Promise.all( | ||
| workflow.steps.map( | ||
| (step) => fs.copy(src, path.join(step.lambda, cmaFolder)) | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * Delete cumulus message adapter from all tasks in the test workflow | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {string} cmaFolder - the name of the folder where CMA is copied to | ||
| * @returns {Promise.<Array>} an array of undefined values | ||
| */ | ||
| function deleteCMAFromTasks(workflow, cmaFolder) { | ||
| return Promise.all( | ||
| workflow.steps.map( | ||
| (step) => fs.remove(path.join(step.lambda, cmaFolder)) | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * Build a cumulus message for a given workflow | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {Object} configOverride - a cumulus config override object | ||
| * @param {Array} cfOutputs - mocked outputs of a CloudFormation template | ||
| * @returns {Object} the generated cumulus message | ||
| */ | ||
| function messageBuilder(workflow, configOverride, cfOutputs) { | ||
| const workflowConfigs = {}; | ||
| workflow.steps.forEach((step) => { | ||
| workflowConfigs[step.name] = step.cumulusConfig; | ||
| }); | ||
|
|
||
| const config = { | ||
| stack: 'somestack', | ||
| workflowConfigs: { | ||
| [workflow.name]: workflowConfigs | ||
| } | ||
| }; | ||
| Object.assign(config, configOverride); | ||
| config.stackName = config.stack; | ||
|
|
||
| const message = template(workflow.name, { States: workflowConfigs }, config, cfOutputs); | ||
| message.cumulus_meta.message_source = 'local'; | ||
| return message; | ||
| } | ||
|
|
||
| /** | ||
| * Runs a given workflow step (task) | ||
| * | ||
| * @param {string} lambdaPath - the local path to the task (e.g. path/to/task) | ||
| * @param {string} lambdaHandler - the lambda handler (e.g. index.hanlder) | ||
| * @param {Object} message - the cumulus message input for the task | ||
| * @param {string} stepName - name of the step/task | ||
| * @returns {Promise.<Object>} the cumulus message returned by the task | ||
| */ | ||
| async function runStep(lambdaPath, lambdaHandler, message, stepName) { | ||
| const taskFullPath = path.join(process.cwd(), lambdaPath); | ||
| const src = path.join(taskFullPath, 'adapter.zip'); | ||
| const dest = path.join(taskFullPath, 'cumulus-message-adapter'); | ||
|
|
||
| process.env.CUMULUS_MESSAGE_ADAPTER_DIR = dest; | ||
|
|
||
| // add step name to the message | ||
| message.cumulus_meta.task = stepName; | ||
|
|
||
| try { | ||
| // run the task | ||
| const moduleFn = lambdaHandler.split('.'); | ||
| const moduleFileName = moduleFn[0]; | ||
| const moduleFunctionName = moduleFn[1]; | ||
| const task = require(`${taskFullPath}/${moduleFileName}`); // eslint-disable-line global-require | ||
|
|
||
| console.log(`Started execution of ${stepName}`); | ||
|
|
||
| return new Promise((resolve, reject) => { | ||
| task[moduleFunctionName](message, {}, (e, r) => { | ||
| if (e) return reject(e); | ||
| console.log(`Completed execution of ${stepName}`); | ||
| return resolve(r); | ||
| }); | ||
| }); | ||
| } | ||
| finally { | ||
| await fs.remove(src); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Executes a given workflow by running each step in the workflow | ||
| * one after each other | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {Object} message - input message to the workflow | ||
| * @returns {Promise.<Object>} an object that includes the workflow input/output | ||
| * plus the output of every step | ||
| */ | ||
| async function runWorkflow(workflow, message) { | ||
| const trail = { | ||
| input: clone(message), | ||
| stepOutputs: {}, | ||
| output: {} | ||
| }; | ||
|
|
||
| let stepInput = clone(message); | ||
|
|
||
| for (const step of workflow.steps) { | ||
| stepInput = await runStep(step.lambda, step.handler, stepInput, step.name); | ||
| trail.stepOutputs[step.name] = clone(stepInput); | ||
| } | ||
| trail.output = clone(stepInput); | ||
|
|
||
| return trail; | ||
| } | ||
|
|
||
| module.exports = { | ||
| downloadCMA, | ||
| copyCMAToTasks, | ||
| deleteCMAFromTasks, | ||
| runStep, | ||
| runWorkflow, | ||
| messageBuilder | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| { | ||
| "MOD09GQ": { | ||
| "name": "MOD09GQ", | ||
| "version": "006", | ||
| "dataType": "MOD09GQ", | ||
| "process": "modis", | ||
| "provider_path": "/pdrs", | ||
| "granuleId": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}$", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", | ||
| "granuleIdExtraction": "(MOD09GQ\\.(.*))\\.hdf", | ||
| "files": [ | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}\\.hdf$", | ||
| "bucket": "protected", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf" | ||
| }, | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}\\.hdf\\.met$", | ||
| "bucket": "private", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf.met" | ||
| }, | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}\\.meta\\.xml$", | ||
| "bucket": "protected", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.meta.xml" | ||
| }, | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}_1\\.jpg$", | ||
| "bucket": "public", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104_1.jpg" | ||
| } | ||
| ] | ||
| } | ||
| } |
Oops, something went wrong.