From b51c56628418b28965236f6bd0f64432d41fcb91 Mon Sep 17 00:00:00 2001 From: Lauren Frederick Date: Tue, 27 Feb 2018 09:33:53 -0500 Subject: [PATCH 1/7] CUMULUS-200 Workflow test runner --- packages/integration-tests/bin/cli.js | 26 +++ packages/integration-tests/index.js | 169 +++++++++++++++++++ packages/integration-tests/package.json | 40 +++++ packages/integration-tests/webpack.config.js | 22 +++ 4 files changed, 257 insertions(+) create mode 100755 packages/integration-tests/bin/cli.js create mode 100644 packages/integration-tests/index.js create mode 100644 packages/integration-tests/package.json create mode 100644 packages/integration-tests/webpack.config.js diff --git a/packages/integration-tests/bin/cli.js b/packages/integration-tests/bin/cli.js new file mode 100755 index 00000000000..4d366dad473 --- /dev/null +++ b/packages/integration-tests/bin/cli.js @@ -0,0 +1,26 @@ +#!/usr/bin/env node + +'use strict'; + +const pckg = require('../package.json'); +const testRunner = require('../index'); +const program = require('commander'); + +program.version(pckg.version); + +program + .usage('TYPE COMMAND [options]') + .option('-s, --stack-name ', 'AWS Cloud Formation stack name', null) + .option('-b, --bucket-name ', 'AWS S3 internal bucket name', null) + .option('-w, --workflow ', 'Workflow name', null) + .option('-i, --input-file ', 'Workflow input JSON file', null); + +program + .command('workflow') + .action(() => { + testRunner.testWorkflow(program.stackName, program.bucketName, + program.workflow, program.inputFile); + }); + +program + .parse(process.argv); diff --git a/packages/integration-tests/index.js b/packages/integration-tests/index.js new file mode 100644 index 00000000000..c71e22d7766 --- /dev/null +++ b/packages/integration-tests/index.js @@ -0,0 +1,169 @@ +'use strict'; + +const aws = require('@cumulus/common/aws'); +const uuidv4 = require('uuid/v4'); + +const waitPeriodMs = 5000; + +/** + * Wait for the defined number of milliseconds + * + * @returns {*} none + */ +function timeout() { + return new Promise((resolve) => setTimeout(resolve, waitPeriodMs)); +} + +/** + * Get the list of workflows for the stack + * + * @param {string} stackName - Cloud formation stack name + * @param {string} bucketName - S3 internal bucket name + * @returns {Object} list as a JSON object + */ +async function getWorkflowList(stackName, bucketName) { + const key = `${stackName}/workflows/list.json`; + + const workflowJson = await aws.s3().getObject({ Bucket: bucketName, Key: key }).promise(); + + return JSON.parse(workflowJson.Body.toString()); +} + +/** + * Get the template JSON from S3 for the workflow + * + * @param {string} stackName - Cloud formation stack name + * @param {string} bucketName - S3 internal bucket name + * @param {string} workflowName - workflow name + * @returns {Object} template as a JSON object + */ +async function getWorkflowTemplate(stackName, bucketName, workflowName) { + const key = `${stackName}/workflows/${workflowName}.json`; + const templateJson = await aws.s3().getObject({ Bucket: bucketName, Key: key }).promise(); + + return JSON.parse(templateJson.Body.toString()); +} + +/** + * Get the workflow ARN for the given workflow from the + * template stored on S3 + * + * @param {string} stackName - Cloud formation stack name + * @param {string} bucketName - S3 internal bucket name + * @param {string} workflowName - workflow name + * @returns {string} - workflow arn + */ +async function getWorkflowArn(stackName, bucketName, workflowName) { + const template = await getWorkflowTemplate(stackName, bucketName, workflowName); + + return template.cumulus_meta.state_machine; +} + +/** + * Get the execution status (i.e. running, completed, etc) + * for the given execution + * + * @param {string} executionArn - ARN of the execution + * @returns {string} status + */ +async function getExecutionStatus(executionArn) { + const status = await aws.sfn().describeExecution({ + executionArn: executionArn + }).promise(); + + return status.status; +} + +/** + * Wait for a given execution to complete, then return the status + * + * @param {string} executionArn - ARN of the execution + * @returns {string} status + */ +async function completeExecution(executionArn) { + // Initialize status to running + let executionStatus = 'RUNNING'; + + // While execution is running, check status on a time interval + while (executionStatus === 'RUNNING') { + await timeout(); + executionStatus = await getExecutionStatus(executionArn); + } + + return executionStatus; +} + +/** + * Kick off a workflow execution + * + * @param {string} workflowArn - ARN for the workflow + * @param {string} inputFile - path to input JSON + * @returns {Object} execution details: {executionArn, startDate} + */ +async function startWorkflowExecution(workflowArn, inputFile) { + const message = require(inputFile); + + // Give this execution a unique name + message.cumulus_meta.execution_name = uuidv4(); + message.cumulus_meta.workflow_start_time = null; + + const workflowParams = { + stateMachineArn: workflowArn, + input: JSON.stringify(message), + name: message.cumulus_meta.execution_name + }; + + return await aws.sfn().startExecution(workflowParams).promise(); +} + +/** + * Execute the given workflow. + * Wait for workflow to complete to get the status + * Return the execution arn and the workflow status. + * + * @param {string} stackName - Cloud formation stack name + * @param {string} bucketName - S3 internal bucket name + * @param {string} workflowName - workflow name + * @param {string} inputFile - path to input JSON file + * @returns {Object} - {executionArn: , status: } + */ +async function executeWorkflow(stackName, bucketName, workflowName, inputFile) { + const workflowArn = await getWorkflowArn(stackName, bucketName, workflowName); + const execution = await startWorkflowExecution(workflowArn, inputFile); + + console.log('Executing workflow: ' + workflowName + + ' . Execution ARN: ' + execution.executionArn); + + // Wait for the execution to complete to get the status + const status = await completeExecution(execution.executionArn); + + return { status: status, arn: execution.executionArn }; +} + +/** + * Test the given workflow and report whether the workflow failed or succeeded + * + * @param {string} stackName - Cloud formation stack name + * @param {string} bucketName - S3 internal bucket name + * @param {string} workflowName - workflow name + * @param {string} inputFile - path to input JSON file + * @returns {*} none + */ +async function testWorkflow(stackName, bucketName, workflowName, inputFile) { + try { + const workflowStatus = await executeWorkflow(stackName, bucketName, workflowName, inputFile); + + if (workflowStatus.status === 'SUCCEEDED') { + console.log('Workflow ' + workflowName + ' execution succeeded.'); + } + else { + console.log('Workflow ' + workflowName + + ' execution failed with state: ' + workflowStatus.status); + } + } + catch (err) { + console.log('Error in workflow ' + workflowName + ' ' + err); + } +} + +exports.testWorkflow = testWorkflow; diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json new file mode 100644 index 00000000000..b9719946f57 --- /dev/null +++ b/packages/integration-tests/package.json @@ -0,0 +1,40 @@ +{ + "name": "@cumulus/integration-tests", + "version": "1.0.0", + "description": "Integration tests", + "bin": { + "cumulus-test": "./bin/cli.js" + }, + "repository": { + "type": "git", + "url": "https://github.com/cumulus-nasa/cumulus" + }, + "scripts": { + "build": "webpack --progress", + "watch": "webpack --progress -w" + }, + "publishConfig": { + "access": "public" + }, + "babel": { + "presets": [ + "es2017" + ], + "plugins": [ + "transform-async-to-generator" + ] + }, + "author": "Cumulus Authors", + "license": "Apache-2.0", + "dependencies": { + "@cumulus/common": "^1.0.0-beta.19", + "babel-core": "^6.25.0", + "babel-loader": "^6.2.4", + "babel-plugin-transform-async-to-generator": "^6.24.1", + "babel-polyfill": "^6.23.0", + "babel-preset-es2017": "^6.24.1", + "commander": "^2.9.0", + "uuid": "^3.2.1", + "webpack": "^1.12.13" + } +} diff --git a/packages/integration-tests/webpack.config.js b/packages/integration-tests/webpack.config.js new file mode 100644 index 00000000000..3d163ff19ed --- /dev/null +++ b/packages/integration-tests/webpack.config.js @@ -0,0 +1,22 @@ +module.exports = { + entry: ['babel-polyfill', './index.js'], + output: { + libraryTarget: 'commonjs2', + filename: 'dist/index.js' + }, + externals: [ + 'electron' + ], + target: 'node', + devtool: 'sourcemap', + module: { + loaders: [{ + test: /\.js?$/, + exclude: /node_modules(?!\/@cumulus\/)/, + loader: 'babel' + }, { + test: /\.json$/, + loader: 'json' + }] + } +}; \ No newline at end of file From 4e76c33dbac01c2d9ad0990803d5aa2a71b253b4 Mon Sep 17 00:00:00 2001 From: Lauren Frederick Date: Tue, 27 Feb 2018 11:17:52 -0500 Subject: [PATCH 2/7] CUMULUS-200 Add eslint JSON, make some updates --- packages/integration-tests/.eslintrc.json | 5 ++++ packages/integration-tests/index.js | 36 ++++++++++++----------- packages/integration-tests/package.json | 1 + 3 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 packages/integration-tests/.eslintrc.json diff --git a/packages/integration-tests/.eslintrc.json b/packages/integration-tests/.eslintrc.json new file mode 100644 index 00000000000..34f481ae722 --- /dev/null +++ b/packages/integration-tests/.eslintrc.json @@ -0,0 +1,5 @@ +{ + "rules": { + "no-console": "off" + } +} \ No newline at end of file diff --git a/packages/integration-tests/index.js b/packages/integration-tests/index.js index c71e22d7766..0aeafb25a28 100644 --- a/packages/integration-tests/index.js +++ b/packages/integration-tests/index.js @@ -2,16 +2,18 @@ const aws = require('@cumulus/common/aws'); const uuidv4 = require('uuid/v4'); +const fs = require('fs-extra'); const waitPeriodMs = 5000; /** * Wait for the defined number of milliseconds * - * @returns {*} none + * @param {integer} waitPeriod - number of milliseconds to wait + * @returns {Promise} ????? */ -function timeout() { - return new Promise((resolve) => setTimeout(resolve, waitPeriodMs)); +function timeout(waitPeriod) { + return new Promise((resolve) => setTimeout(resolve, waitPeriod)); } /** @@ -80,13 +82,12 @@ async function getExecutionStatus(executionArn) { * @param {string} executionArn - ARN of the execution * @returns {string} status */ -async function completeExecution(executionArn) { - // Initialize status to running - let executionStatus = 'RUNNING'; +async function waitForCompletedExecution(executionArn) { + let executionStatus = await getExecutionStatus(executionArn); // While execution is running, check status on a time interval while (executionStatus === 'RUNNING') { - await timeout(); + await timeout(waitPeriodMs); executionStatus = await getExecutionStatus(executionArn); } @@ -101,19 +102,21 @@ async function completeExecution(executionArn) { * @returns {Object} execution details: {executionArn, startDate} */ async function startWorkflowExecution(workflowArn, inputFile) { - const message = require(inputFile); + const rawInput = await fs.readFile(inputFile, 'utf8'); + + const parsedInput = JSON.parse(rawInput); // Give this execution a unique name - message.cumulus_meta.execution_name = uuidv4(); - message.cumulus_meta.workflow_start_time = null; + parsedInput.cumulus_meta.execution_name = uuidv4(); + parsedInput.cumulus_meta.workflow_start_time = null; const workflowParams = { stateMachineArn: workflowArn, - input: JSON.stringify(message), - name: message.cumulus_meta.execution_name + input: JSON.stringify(parsedInput), + name: parsedInput.cumulus_meta.execution_name }; - return await aws.sfn().startExecution(workflowParams).promise(); + return aws.sfn().startExecution(workflowParams).promise(); } /** @@ -131,11 +134,10 @@ async function executeWorkflow(stackName, bucketName, workflowName, inputFile) { const workflowArn = await getWorkflowArn(stackName, bucketName, workflowName); const execution = await startWorkflowExecution(workflowArn, inputFile); - console.log('Executing workflow: ' + workflowName + - ' . Execution ARN: ' + execution.executionArn); + console.log(`Executing workflow: ${workflowName}. Execution ARN ${execution.executionArn}`); // Wait for the execution to complete to get the status - const status = await completeExecution(execution.executionArn); + const status = await waitForCompletedExecution(execution.executionArn); return { status: status, arn: execution.executionArn }; } @@ -162,7 +164,7 @@ async function testWorkflow(stackName, bucketName, workflowName, inputFile) { } } catch (err) { - console.log('Error in workflow ' + workflowName + ' ' + err); + console.log('Error executing workflow ' + workflowName + ' ' + err); } } diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index b9719946f57..7eb522476d3 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -34,6 +34,7 @@ "babel-polyfill": "^6.23.0", "babel-preset-es2017": "^6.24.1", "commander": "^2.9.0", + "fs-extra": "^5.0.0", "uuid": "^3.2.1", "webpack": "^1.12.13" } From 458acf69e49d6ee60a24621a581a6e3dda5b11f4 Mon Sep 17 00:00:00 2001 From: Lauren Frederick Date: Tue, 27 Feb 2018 14:36:16 -0500 Subject: [PATCH 3/7] CUMULUS-200 Updates for review --- packages/integration-tests/index.js | 55 +++++++++++++++-------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/packages/integration-tests/index.js b/packages/integration-tests/index.js index 0aeafb25a28..1b10259dc09 100644 --- a/packages/integration-tests/index.js +++ b/packages/integration-tests/index.js @@ -1,16 +1,17 @@ 'use strict'; -const aws = require('@cumulus/common/aws'); const uuidv4 = require('uuid/v4'); const fs = require('fs-extra'); +const { s3, sfn } = require('@cumulus/common/aws'); +const executionStatusNumRetries = 20; const waitPeriodMs = 5000; /** * Wait for the defined number of milliseconds * * @param {integer} waitPeriod - number of milliseconds to wait - * @returns {Promise} ????? + * @returns {Promise.} - promise resolves after a given time period */ function timeout(waitPeriod) { return new Promise((resolve) => setTimeout(resolve, waitPeriod)); @@ -26,9 +27,8 @@ function timeout(waitPeriod) { async function getWorkflowList(stackName, bucketName) { const key = `${stackName}/workflows/list.json`; - const workflowJson = await aws.s3().getObject({ Bucket: bucketName, Key: key }).promise(); - - return JSON.parse(workflowJson.Body.toString()); + return s3().getObject({ Bucket: bucketName, Key: key }).promise() + .then((workflowJson) => JSON.parse(workflowJson.Body.toString())); } /** @@ -41,9 +41,8 @@ async function getWorkflowList(stackName, bucketName) { */ async function getWorkflowTemplate(stackName, bucketName, workflowName) { const key = `${stackName}/workflows/${workflowName}.json`; - const templateJson = await aws.s3().getObject({ Bucket: bucketName, Key: key }).promise(); - - return JSON.parse(templateJson.Body.toString()); + return s3().getObject({ Bucket: bucketName, Key: key }).promise() + .then((templateJson) => JSON.parse(templateJson.Body.toString())); } /** @@ -56,9 +55,8 @@ async function getWorkflowTemplate(stackName, bucketName, workflowName) { * @returns {string} - workflow arn */ async function getWorkflowArn(stackName, bucketName, workflowName) { - const template = await getWorkflowTemplate(stackName, bucketName, workflowName); - - return template.cumulus_meta.state_machine; + return getWorkflowTemplate(stackName, bucketName, workflowName) + .then((template) => template.cumulus_meta.state_machine); } /** @@ -69,11 +67,8 @@ async function getWorkflowArn(stackName, bucketName, workflowName) { * @returns {string} status */ async function getExecutionStatus(executionArn) { - const status = await aws.sfn().describeExecution({ - executionArn: executionArn - }).promise(); - - return status.status; + return sfn().describeExecution({ executionArn }).promise() + .then((status) => status.status); } /** @@ -84,11 +79,17 @@ async function getExecutionStatus(executionArn) { */ async function waitForCompletedExecution(executionArn) { let executionStatus = await getExecutionStatus(executionArn); + let statusCheckCount = 0; // While execution is running, check status on a time interval - while (executionStatus === 'RUNNING') { + while (executionStatus === 'RUNNING' && statusCheckCount < executionStatusNumRetries) { await timeout(waitPeriodMs); executionStatus = await getExecutionStatus(executionArn); + statusCheckCount++; + } + + if (executionStatus === 'RUNNING' && statusCheckCount === executionStatusNumRetries) { + console.log(`Execution status check timed out, exceeded ${executionStatusNumRetries} status checks.`); } return executionStatus; @@ -99,7 +100,7 @@ async function waitForCompletedExecution(executionArn) { * * @param {string} workflowArn - ARN for the workflow * @param {string} inputFile - path to input JSON - * @returns {Object} execution details: {executionArn, startDate} + * @returns {Promise.} execution details: {executionArn, startDate} */ async function startWorkflowExecution(workflowArn, inputFile) { const rawInput = await fs.readFile(inputFile, 'utf8'); @@ -116,7 +117,7 @@ async function startWorkflowExecution(workflowArn, inputFile) { name: parsedInput.cumulus_meta.execution_name }; - return aws.sfn().startExecution(workflowParams).promise(); + return sfn().startExecution(workflowParams).promise(); } /** @@ -133,13 +134,14 @@ async function startWorkflowExecution(workflowArn, inputFile) { async function executeWorkflow(stackName, bucketName, workflowName, inputFile) { const workflowArn = await getWorkflowArn(stackName, bucketName, workflowName); const execution = await startWorkflowExecution(workflowArn, inputFile); + const executionArn = execution.executionArn; - console.log(`Executing workflow: ${workflowName}. Execution ARN ${execution.executionArn}`); + console.log(`Executing workflow: ${workflowName}. Execution ARN ${executionArn}`); // Wait for the execution to complete to get the status - const status = await waitForCompletedExecution(execution.executionArn); + const status = await waitForCompletedExecution(executionArn); - return { status: status, arn: execution.executionArn }; + return { status, executionArn }; } /** @@ -149,22 +151,21 @@ async function executeWorkflow(stackName, bucketName, workflowName, inputFile) { * @param {string} bucketName - S3 internal bucket name * @param {string} workflowName - workflow name * @param {string} inputFile - path to input JSON file - * @returns {*} none + * @returns {*} undefined */ async function testWorkflow(stackName, bucketName, workflowName, inputFile) { try { const workflowStatus = await executeWorkflow(stackName, bucketName, workflowName, inputFile); if (workflowStatus.status === 'SUCCEEDED') { - console.log('Workflow ' + workflowName + ' execution succeeded.'); + console.log(`Workflow ${workflowName} execution succeeded.`); } else { - console.log('Workflow ' + workflowName + - ' execution failed with state: ' + workflowStatus.status); + console.log(`Workflow ${workflowName} execution failed with state: ${workflowStatus.status}`); } } catch (err) { - console.log('Error executing workflow ' + workflowName + ' ' + err); + console.log(`Error executing workflow ${workflowName}. Error: ${err}`); } } From f29302019af2002c9c16db735ad1b35e11245bca Mon Sep 17 00:00:00 2001 From: Lauren Frederick Date: Tue, 27 Feb 2018 14:46:56 -0500 Subject: [PATCH 4/7] CUMULUS-200 Add workflow command description --- packages/integration-tests/bin/cli.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/integration-tests/bin/cli.js b/packages/integration-tests/bin/cli.js index 4d366dad473..69d3c957c82 100755 --- a/packages/integration-tests/bin/cli.js +++ b/packages/integration-tests/bin/cli.js @@ -17,6 +17,7 @@ program program .command('workflow') + .description('Execute a workflow and determine if the workflow completes successfully') .action(() => { testRunner.testWorkflow(program.stackName, program.bucketName, program.workflow, program.inputFile); From 39a2a6494c8abb6e152d6360134926c289e88df7 Mon Sep 17 00:00:00 2001 From: laurenfrederick Date: Tue, 27 Feb 2018 14:47:56 -0500 Subject: [PATCH 5/7] Create README.md --- packages/integration-tests/README.md | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 packages/integration-tests/README.md diff --git a/packages/integration-tests/README.md b/packages/integration-tests/README.md new file mode 100644 index 00000000000..6a8d1330a44 --- /dev/null +++ b/packages/integration-tests/README.md @@ -0,0 +1,47 @@ +# @cumulus/integration-tests + +[![CircleCI](https://circleci.com/gh/cumulus-nasa/cumulus.svg?style=svg)](https://circleci.com/gh/cumulus-nasa/cumulus) + +@cumulus/integration-tests provides a CLI and functions for testing Cumulus workflow executions in a Cumulus deployment. + +## What is Cumulus? + +Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams. + +[Cumulus Documentation](https://cumulus-nasa.github.io/) + +## Installation + +``` +npm install @cumulus/integration-tests +``` + +## Usage + +``` +Usage: cumulus-test TYPE COMMAND [options] + + + Options: + + -V, --version output the version number + -s, --stack-name AWS Cloud Formation stack name (default: null) + -b, --bucket-name AWS S3 internal bucket name (default: null) + -w, --workflow Workflow name (default: null) + -i, --input-file Workflow input JSON file (default: null) + -h, --help output usage information + + + Commands: + + workflow Execute a workflow and determine if the workflow completes successfully +``` +i.e. to test the HelloWorld workflow: + +`cumulus-test workflow --stack-name helloworld-cumulus --bucket-name cumulus-bucket-internal --workflow HelloWorldWorkflow --input-file ./helloWorldInput.json` + + + +## Contributing + +See [Cumulus README](https://github.com/cumulus-nasa/cumulus/blob/master/README.md#installing-and-deploying) From a90e95299e39a963b8d4f4c5c87440d9e29cd031 Mon Sep 17 00:00:00 2001 From: Lauren Frederick Date: Wed, 28 Feb 2018 12:03:56 -0500 Subject: [PATCH 6/7] CUMULUS-200 PR review fixes --- packages/integration-tests/bin/cli.js | 34 +++++++++++++++++++++++++-- packages/integration-tests/index.js | 29 +++++++---------------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/packages/integration-tests/bin/cli.js b/packages/integration-tests/bin/cli.js index 69d3c957c82..f0be0d62b25 100755 --- a/packages/integration-tests/bin/cli.js +++ b/packages/integration-tests/bin/cli.js @@ -8,6 +8,31 @@ const program = require('commander'); program.version(pckg.version); +/** + * Verify that the given param is not null. Write out an error if null. + * + * @param {Object} paramConfig - param name and value {name: value:} + * @returns {boolean} true if param is not null + */ +function verifyRequiredParameter(paramConfig) { + if (paramConfig.value === null) { + console.log(`Error: ${paramConfig.name} is a required parameter.`); + return false; + } + + return true; +} + +/** + * Verify required parameters are present + * + * @param {list} requiredParams - params in the form {name: 'x' value: 'y'} + * @returns {boolean} - true if all params are not null + */ +function verifyWorkflowParameters(requiredParams) { + return requiredParams.map((param) => verifyRequiredParameter(param)).includes(false) === false; +} + program .usage('TYPE COMMAND [options]') .option('-s, --stack-name ', 'AWS Cloud Formation stack name', null) @@ -19,8 +44,13 @@ program .command('workflow') .description('Execute a workflow and determine if the workflow completes successfully') .action(() => { - testRunner.testWorkflow(program.stackName, program.bucketName, - program.workflow, program.inputFile); + if (verifyWorkflowParameters([{ name: 'stack-name', value: program.stackName }, + { name: 'bucket-name', value: program.bucketName }, + { name: 'workflow', value: program.workflow }, + { name: 'input-file', value: program.inputFile }])) { + testRunner.testWorkflow(program.stackName, program.bucketName, + program.workflow, program.inputFile); + } }); program diff --git a/packages/integration-tests/index.js b/packages/integration-tests/index.js index 1b10259dc09..6a8432eaf55 100644 --- a/packages/integration-tests/index.js +++ b/packages/integration-tests/index.js @@ -17,27 +17,13 @@ function timeout(waitPeriod) { return new Promise((resolve) => setTimeout(resolve, waitPeriod)); } -/** - * Get the list of workflows for the stack - * - * @param {string} stackName - Cloud formation stack name - * @param {string} bucketName - S3 internal bucket name - * @returns {Object} list as a JSON object - */ -async function getWorkflowList(stackName, bucketName) { - const key = `${stackName}/workflows/list.json`; - - return s3().getObject({ Bucket: bucketName, Key: key }).promise() - .then((workflowJson) => JSON.parse(workflowJson.Body.toString())); -} - /** * Get the template JSON from S3 for the workflow * * @param {string} stackName - Cloud formation stack name * @param {string} bucketName - S3 internal bucket name * @param {string} workflowName - workflow name - * @returns {Object} template as a JSON object + * @returns {Promise.} template as a JSON object */ async function getWorkflowTemplate(stackName, bucketName, workflowName) { const key = `${stackName}/workflows/${workflowName}.json`; @@ -52,9 +38,9 @@ async function getWorkflowTemplate(stackName, bucketName, workflowName) { * @param {string} stackName - Cloud formation stack name * @param {string} bucketName - S3 internal bucket name * @param {string} workflowName - workflow name - * @returns {string} - workflow arn + * @returns {Promise.} - workflow arn */ -async function getWorkflowArn(stackName, bucketName, workflowName) { +function getWorkflowArn(stackName, bucketName, workflowName) { return getWorkflowTemplate(stackName, bucketName, workflowName) .then((template) => template.cumulus_meta.state_machine); } @@ -66,7 +52,7 @@ async function getWorkflowArn(stackName, bucketName, workflowName) { * @param {string} executionArn - ARN of the execution * @returns {string} status */ -async function getExecutionStatus(executionArn) { +function getExecutionStatus(executionArn) { return sfn().describeExecution({ executionArn }).promise() .then((status) => status.status); } @@ -88,7 +74,8 @@ async function waitForCompletedExecution(executionArn) { statusCheckCount++; } - if (executionStatus === 'RUNNING' && statusCheckCount === executionStatusNumRetries) { + if (executionStatus === 'RUNNING' && statusCheckCount >= executionStatusNumRetries) { + //eslint-disable-next-line max-len console.log(`Execution status check timed out, exceeded ${executionStatusNumRetries} status checks.`); } @@ -109,7 +96,8 @@ async function startWorkflowExecution(workflowArn, inputFile) { // Give this execution a unique name parsedInput.cumulus_meta.execution_name = uuidv4(); - parsedInput.cumulus_meta.workflow_start_time = null; + parsedInput.cumulus_meta.workflow_start_time = Date.now(); + parsedInput.cumulus_meta.state_machine = workflowArn; const workflowParams = { stateMachineArn: workflowArn, @@ -170,3 +158,4 @@ async function testWorkflow(stackName, bucketName, workflowName, inputFile) { } exports.testWorkflow = testWorkflow; +exports.executeWorkflow = executeWorkflow; From 7e9dff84b448b1a50e9be6819816bd3b4cf41afc Mon Sep 17 00:00:00 2001 From: Lauren Frederick Date: Wed, 28 Feb 2018 15:49:56 -0500 Subject: [PATCH 7/7] CUMULUS-200 more pr changes --- packages/integration-tests/bin/cli.js | 2 +- packages/integration-tests/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/integration-tests/bin/cli.js b/packages/integration-tests/bin/cli.js index f0be0d62b25..54e38a1d026 100755 --- a/packages/integration-tests/bin/cli.js +++ b/packages/integration-tests/bin/cli.js @@ -30,7 +30,7 @@ function verifyRequiredParameter(paramConfig) { * @returns {boolean} - true if all params are not null */ function verifyWorkflowParameters(requiredParams) { - return requiredParams.map((param) => verifyRequiredParameter(param)).includes(false) === false; + return requiredParams.map(verifyRequiredParameter).includes(false) === false; } program diff --git a/packages/integration-tests/index.js b/packages/integration-tests/index.js index 6a8432eaf55..1cc89631642 100644 --- a/packages/integration-tests/index.js +++ b/packages/integration-tests/index.js @@ -25,7 +25,7 @@ function timeout(waitPeriod) { * @param {string} workflowName - workflow name * @returns {Promise.} template as a JSON object */ -async function getWorkflowTemplate(stackName, bucketName, workflowName) { +function getWorkflowTemplate(stackName, bucketName, workflowName) { const key = `${stackName}/workflows/${workflowName}.json`; return s3().getObject({ Bucket: bucketName, Key: key }).promise() .then((templateJson) => JSON.parse(templateJson.Body.toString()));