end-to-end tests in circleci [CUMULUS-371] #238
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| /** | ||
| * Includes helper functions for replicating Step Function Workflows | ||
| * locally | ||
| */ | ||
| 'use strict'; | ||
|
|
||
| const path = require('path'); | ||
| const fs = require('fs-extra'); | ||
| const clone = require('lodash.clonedeep'); | ||
| const { randomString } = require('@cumulus/common/test-utils'); | ||
| const { template } = require('@cumulus/deployment/lib/message'); | ||
| const { fetchMessageAdapter } = require('@cumulus/deployment/lib/adapter'); | ||
|
|
||
| /** | ||
| * Download cumulus message adapter (CMA) and unzip it | ||
| * | ||
| * @param {string} version - cumulus message adapter version number (optional) | ||
| * @returns {Promise.<Object>} an object with path to the zip and extracted CMA | ||
| */ | ||
| async function downloadCMA(version) { | ||
| // download and unzip the message adapter | ||
| const gitPath = 'cumulus-nasa/cumulus-message-adapter'; | ||
| const filename = 'cumulus-message-adapter.zip'; | ||
| const src = path.join(process.cwd(), 'tests', `${randomString()}.zip`); | ||
| const dest = path.join(process.cwd(), 'tests', randomString()); | ||
| await fetchMessageAdapter(version, gitPath, filename, src, dest); | ||
| return { | ||
| src, | ||
| dest | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Copy cumulus message adapter python folder to each task | ||
| * in the workflow | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {string} src - the path to the cumulus message adapter folder | ||
| * @param {string} cmaFolder - the name of the folder where CMA is copied to | ||
| * @returns {Promise.<Array>} an array of undefined values | ||
| */ | ||
| function copyCMAToTasks(workflow, src, cmaFolder) { | ||
| return Promise.all( | ||
| workflow.steps.map( | ||
| (step) => fs.copy(src, path.join(step.lambda, cmaFolder)) | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * Delete cumulus message adapter from all tasks in the test workflow | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {string} cmaFolder - the name of the folder where CMA is copied to | ||
| * @returns {Promise.<Array>} an array of undefined values | ||
| */ | ||
| function deleteCMAFromTasks(workflow, cmaFolder) { | ||
| return Promise.all( | ||
| workflow.steps.map( | ||
| (step) => fs.remove(path.join(step.lambda, cmaFolder)) | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * Build a cumulus message for a given workflow | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {Object} configOverride - a cumulus config override object | ||
| * @param {Array} cfOutputs - mocked outputs of a CloudFormation template | ||
| * @returns {Object} the generated cumulus message | ||
| */ | ||
| function messageBuilder(workflow, configOverride, cfOutputs) { | ||
|
There was a problem hiding this comment. Non-blocking but here and elsewhere in this PR it could be more maintainable to use an object argument, e.g. this makes it easier to add arguments without being concerned about what order they are in when you make the function call. |
||
| const workflowConfigs = {}; | ||
| workflow.steps.forEach((step) => { | ||
| workflowConfigs[step.name] = step.cumulusConfig; | ||
| }); | ||
|
|
||
| const config = { | ||
| stack: 'somestack', | ||
| workflowConfigs: { | ||
| [workflow.name]: workflowConfigs | ||
| } | ||
| }; | ||
| Object.assign(config, configOverride); | ||
| config.stackName = config.stack; | ||
|
|
||
| const message = template(workflow.name, { States: workflowConfigs }, config, cfOutputs); | ||
| message.cumulus_meta.message_source = 'local'; | ||
| return message; | ||
| } | ||
|
|
||
| /** | ||
| * Runs a given workflow step (task) | ||
| * | ||
| * @param {string} lambdaPath - the local path to the task (e.g. path/to/task) | ||
| * @param {string} lambdaHandler - the lambda handler (e.g. index.hanlder) | ||
| * @param {Object} message - the cumulus message input for the task | ||
| * @param {string} stepName - name of the step/task | ||
| * @returns {Promise.<Object>} the cumulus message returned by the task | ||
| */ | ||
| async function runStep(lambdaPath, lambdaHandler, message, stepName) { | ||
| const taskFullPath = path.join(process.cwd(), lambdaPath); | ||
| const src = path.join(taskFullPath, 'adapter.zip'); | ||
| const dest = path.join(taskFullPath, 'cumulus-message-adapter'); | ||
|
|
||
| process.env.CUMULUS_MESSAGE_ADAPTER_DIR = dest; | ||
|
|
||
| // add step name to the message | ||
| message.cumulus_meta.task = stepName; | ||
|
|
||
| try { | ||
| // run the task | ||
| const moduleFn = lambdaHandler.split('.'); | ||
| const moduleFileName = moduleFn[0]; | ||
| const moduleFunctionName = moduleFn[1]; | ||
| const task = require(`${taskFullPath}/${moduleFileName}`); // eslint-disable-line global-require | ||
|
|
||
| console.log(`Started execution of ${stepName}`); | ||
|
|
||
| return new Promise((resolve, reject) => { | ||
| task[moduleFunctionName](message, {}, (e, r) => { | ||
| if (e) return reject(e); | ||
| console.log(`Completed execution of ${stepName}`); | ||
| return resolve(r); | ||
|
There was a problem hiding this comment. I don't think a There was a problem hiding this comment. it's good practice to return something in a function if possible. |
||
| }); | ||
| }); | ||
| } | ||
| finally { | ||
| await fs.remove(src); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Executes a given workflow by running each step in the workflow | ||
| * one after each other | ||
| * | ||
| * @param {Object} workflow - a test workflow object | ||
| * @param {Object} message - input message to the workflow | ||
| * @returns {Promise.<Object>} an object that includes the workflow input/output | ||
| * plus the output of every step | ||
| */ | ||
| async function runWorkflow(workflow, message) { | ||
| const trail = { | ||
| input: clone(message), | ||
| stepOutputs: {}, | ||
| output: {} | ||
| }; | ||
|
|
||
| let stepInput = clone(message); | ||
|
|
||
| for (const step of workflow.steps) { | ||
|
There was a problem hiding this comment. Curious - is there a reason to use this iteration syntax over There was a problem hiding this comment. This has to happen in sequence. If we don't use for of, we have to use some other library to ensure each async task happen after the other one ended. There was a problem hiding this comment. I see that makes sense |
||
| stepInput = await runStep(step.lambda, step.handler, stepInput, step.name); | ||
| trail.stepOutputs[step.name] = clone(stepInput); | ||
| } | ||
| trail.output = clone(stepInput); | ||
|
|
||
| return trail; | ||
|
There was a problem hiding this comment. I don't think trail is a promise, as documentation suggests is returned from this function but I could be missing something There was a problem hiding this comment. this is an async/await function so it always return a promise. |
||
| } | ||
|
|
||
| module.exports = { | ||
| downloadCMA, | ||
| copyCMAToTasks, | ||
| deleteCMAFromTasks, | ||
| runStep, | ||
| runWorkflow, | ||
| messageBuilder | ||
| }; | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| { | ||
| "MOD09GQ": { | ||
| "name": "MOD09GQ", | ||
| "version": "006", | ||
| "dataType": "MOD09GQ", | ||
| "process": "modis", | ||
| "provider_path": "/pdrs", | ||
| "granuleId": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}$", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf", | ||
| "granuleIdExtraction": "(MOD09GQ\\.(.*))\\.hdf", | ||
| "files": [ | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}\\.hdf$", | ||
| "bucket": "protected", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf" | ||
| }, | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}\\.hdf\\.met$", | ||
| "bucket": "private", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.hdf.met" | ||
| }, | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}\\.meta\\.xml$", | ||
| "bucket": "protected", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104.meta.xml" | ||
| }, | ||
| { | ||
| "regex": "^MOD09GQ\\.A[\\d]{7}\\.[\\S]{6}\\.006.[\\d]{13}_1\\.jpg$", | ||
| "bucket": "public", | ||
| "sampleFileName": "MOD09GQ.A2017025.h21v00.006.2017034065104_1.jpg" | ||
| } | ||
| ] | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think this function just returns an array of messages (or the empty array) (e.g. not
{Promise.<Array>}as documentation above indicatesThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
async/await function always return a promise.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You're right! I learned something.