CUMULUS-436: Cumulus Integration Tests should include an ECS task

app/config.yml

@@ -2,6 +2,13 @@
 default:
   stackName: test-cumulus-integration
   stackNameNoDash: TestCumulusIntegration
+  params:
+    - name: CmrPassword
+      value: '{{CMR_PASSWORD}}'
+    - name: DockerEmail
+      value: '{{DOCKER_EMAIL}}'
+    - name: DockerPassword
+      value: '{{DOCKER_PASS}}'
 
   apiStage: dev
 
@@ -11,17 +18,45 @@ default:
       - '{{AWS_SUBNET}}'
 
   ecs:
-    instanceType: t2.micro
+    instanceType: t2.small
+    desiredInstances: 1
+    availabilityZone: us-east-1a
+    imageId: ami-a7a242da
+    publicIp: true
+    docker:
+      username: cumulususer
+    services:
+      EcsTaskHelloWorld:
+        image: cumuluss/cumulus-ecs-task:1.0.2
+        cpu: 800
+        memory: 1500
+        count: 1
+        envs:
+          AWS_DEFAULT_REGION:
+            function: Fn::Sub
+            value: '${AWS::Region}'
+        commands:
+          - cumulus-ecs-task
+          - '--activityArn'
+          - function: Ref
+            value: EcsTaskHelloWorldActivity
+          - '--lambdaArn'
+          - function: Ref
+            value: HelloWorldLambdaFunction
+
+  activities:
+    - name: EcsTaskHelloWorld
 
   buckets:
     internal: cumulus-test-sandbox-internal
     private: cumulus-test-sandbox-private
 
   iams:
-    ecsRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-ecs'
-    lambdaApiGatewayRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-lambda-api-gateway'
-    lambdaProcessingRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-lambda-processing'
-    stepRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-steprole'
+    ecsRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-integration-ecs'
+    lambdaApiGatewayRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-integration-lambda-api-gateway'
+    lambdaProcessingRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-integration-lambda-processing'
+    stepRoleArn: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/test-cumulus-integration-steprole'
+    instanceProfile: 'arn:aws:iam::{{AWS_ACCOUNT_ID}}:instance-profile/test-cumulus-integration-ecs'
 
   urs_url: https://uat.urs.earthdata.nasa.gov/ #make sure to include the trailing slash
 

iam/cloudformation.template.yml

@@ -359,6 +359,7 @@ Resources:
                 - states:SendTaskSuccess
                 - states:GetActivityTask
                 - states:DescribeActivity
+                - states:GetExecutionHistory
                 Resource: arn:aws:states:*:*:*
 
               # Allow writing to ingest buckets

iam/config.yml

@@ -1,6 +1,6 @@
 default:
-  prefix: test-cumulus-intg
-  stackName: test-cumulus-intg-iam
+  prefix: test-cumulus
+  stackName: test-cumulus-integration-iam
   authorizor: false
   capabilities:
     - CAPABILITY_NAMED_IAM

package.json

@@ -22,23 +22,24 @@
     ]
   },
   "dependencies": {
-    "@cumulus/api": "^1.2.0",
-    "@cumulus/deployment": "^1.2.0",
-    "@cumulus/discover-pdrs": "^1.2.0",
-    "@cumulus/hello-world": "^1.2.0",
-    "@cumulus/integration-tests": "^1.2.0",
-    "@cumulus/parse-pdr": "^1.2.0",
-    "@cumulus/pdr-status-check": "^1.2.0",
-    "@cumulus/queue-granules": "^1.2.0",
-    "@cumulus/queue-pdrs": "^1.2.0",
-    "@cumulus/sync-granule": "^1.2.0",
+    "@cumulus/api": "^1.3.0",
+    "@cumulus/deployment": "^1.3.0",
+    "@cumulus/discover-pdrs": "^1.3.0",
+    "@cumulus/hello-world": "^1.3.0",
+    "@cumulus/integration-tests": "^1.3.0",
+    "@cumulus/parse-pdr": "^1.3.0",
+    "@cumulus/pdr-status-check": "^1.3.0",
+    "@cumulus/queue-granules": "^1.3.0",
+    "@cumulus/queue-pdrs": "^1.3.0",
+    "@cumulus/sync-granule": "^1.3.0",
     "aws-sdk": "^2.209.0",
     "jasmine": "^3.1.0",
     "jasmine-console-reporter": "^2.0.1",
     "js-yaml": "^3.11.0",
     "kes": "^2.1.2",
     "lodash": "^4.17.5",
-    "node-forge": "^0.7.3"
+    "node-forge": "^0.7.3",
+    "webpack": "^1.12.13"
   },
   "devDependencies": {
     "babel-cli": "^6.26.0",

spec/discoverAndQueuePdrs/DiscoverAndQueuePdrs.input.template.json

@@ -11,7 +11,9 @@
       "internal": ""
     },
     "collection": {
-      "provider_path": "cumulus-test-data/pdrs"
+      "provider_path": "cumulus-test-data/pdrs",
+      "name": "test-collection",
+      "granuleIdExtraction": "(.*)"
     },
     "provider": {
       "id": "s3_provider",

spec/discoverAndQueuePdrs/DiscoverAndQueuePdrsSuccessSpec.js

@@ -1,10 +1,12 @@
 const { S3 } = require('aws-sdk');
-const workflow = require('@cumulus/integration-tests');
+const { executeWorkflow, LambdaStep } = require('@cumulus/integration-tests');
 
 const { loadConfig, templateFile } = require('../helpers/testUtils');
 
 const s3 = new S3();
 const config = loadConfig();
+const lambdaStep = new LambdaStep();
+
 const taskName = 'DiscoverAndQueuePdrs';
 const inputTemplateFilename = './spec/discoverAndQueuePdrs/DiscoverAndQueuePdrs.input.template.json';
 const templatedInputFilename = templateFile({
@@ -20,7 +22,7 @@ describe("The Discover And Queue PDRs workflow", function() {
   let workflowExecution = null;
 
   beforeAll(async function() {
-    workflowExecution = await workflow.executeWorkflow(
+    workflowExecution = await executeWorkflow(
       config.stackName,
       config.bucket,
       taskName,
@@ -43,7 +45,7 @@ describe("The Discover And Queue PDRs workflow", function() {
     let lambdaOutput = null;
 
     beforeAll(async function() {
-      lambdaOutput = await workflow.getLambdaOutput(workflowExecution.executionArn, "DiscoverPdrs");
+      lambdaOutput = await lambdaStep.getStepOutput(workflowExecution.executionArn, "DiscoverPdrs");
     });
 
     it("has expected path and name output", function() {
@@ -56,7 +58,7 @@ describe("The Discover And Queue PDRs workflow", function() {
     let lambdaOutput = null;
 
     beforeAll(async function() {
-      lambdaOutput = await workflow.getLambdaOutput(workflowExecution.executionArn, "QueuePdrs");
+      lambdaOutput = await lambdaStep.getStepOutput(workflowExecution.executionArn, "QueuePdrs");
     });
 
     it("output is pdrs_queued", function() {

spec/helloWorld/HelloWorldSuccessSpec.js

@@ -1,14 +1,17 @@
-const workflow = require('@cumulus/integration-tests');
+const { executeWorkflow, LambdaStep, ActivityStep } = require('@cumulus/integration-tests');
 const { loadConfig } = require('../helpers/testUtils');
+
 const awsConfig = loadConfig();
+const lambdaStep = new LambdaStep();
+const activityStep = new ActivityStep();
 
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 120000;
 
-describe("The Hello World workflow", function() {
+describe('The Hello World workflow', function() {
   let workflowExecution = null;
 
   beforeAll(async function() {
-    workflowExecution = await workflow.executeWorkflow(
+    workflowExecution = await executeWorkflow(
       awsConfig.stackName,
       awsConfig.bucket,
       'HelloWorldWorkflow',
@@ -20,15 +23,44 @@ describe("The Hello World workflow", function() {
     expect(workflowExecution.status).toEqual('SUCCEEDED');
   });
 
-  describe("the HelloWorld Lambda", function() {
+  describe('the HelloWorld Lambda', function() {
     let lambdaOutput = null;
 
     beforeAll(async function() {
-      lambdaOutput = await workflow.getLambdaOutput(workflowExecution.executionArn, "HelloWorld");
+      lambdaOutput = await lambdaStep.getStepOutput(workflowExecution.executionArn, 'HelloWorld');
     });
 
-    it("output is Hello World", function() {
+    it('output is Hello World', function() {
       expect(lambdaOutput.payload).toEqual({ hello: 'Hello World' });
     });
   });
-});
+});
+
+describe('The Hello World workflow using ECS', function() {
+  let workflowExecution = null;
+
+  beforeAll(async function() {
+    workflowExecution = await executeWorkflow(
+      awsConfig.stackName,
+      awsConfig.bucket,
+      'EcsHelloWorldWorkflow',
+      'spec/helloWorld/HelloWorld.input.json'
+    );
+  });
+
+  it('executes successfully', function() {
+    expect(workflowExecution.status).toEqual('SUCCEEDED');
+  });
+
+  describe('the HelloWorld ECS', function() {
+    let activityOutput = null;
+
+    beforeAll(async function() {
+      activityOutput = await activityStep.getStepOutput(workflowExecution.executionArn, 'EcsTaskHelloWorld');
+    });
+
+    it('output is Hello World', function() {
+      expect(activityOutput.payload).toEqual({ hello: 'Hello World' });
+    });
+  });
+});

spec/ingestGranule/IngestGranule.input.template.json

@@ -13,7 +13,11 @@
     },
     "collection": {
       "files": [],
-      "provider_path": "cumulus-test-data/pdrs"
+      "provider_path": "cumulus-test-data/pdrs",
+      "name": "test-collection",
+      "granuleIdExtraction": "(.*)",
+      "granuleId": "(.*).hdf",
+      "process": "modis"
     },
     "provider": {
       "host": "cumulus-data-shared",
@@ -49,6 +53,7 @@
   "payload": {
     "granules": [
       {
+        "granuleId": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf",
         "files": [
           {
             "path": "cumulus-test-data/pdrs",

spec/ingestGranule/IngestGranule.output.payload.template.json

@@ -1,6 +1,8 @@
 {
+  "process": "modis",
   "granules": [
     {
+      "granuleId": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf",
       "files": [
         {
           "name": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf",

spec/ingestGranule/IngestGranuleSuccessSpec.js

@@ -1,8 +1,9 @@
 const fs = require('fs');
-const workflow = require('@cumulus/integration-tests');
+const { executeWorkflow, LambdaStep } = require('@cumulus/integration-tests');
 
 const { loadConfig, templateFile } = require('../helpers/testUtils');
 const config = loadConfig();
+const lambdaStep = new LambdaStep();
 const taskName = 'IngestGranule';
 
 const inputTemplateFilename = './spec/ingestGranule/IngestGranule.input.template.json';
@@ -24,7 +25,7 @@ describe("The Ingest Granules workflow", function() {
   let workflowExecution = null;
 
   beforeAll(async function() {
-    workflowExecution = await workflow.executeWorkflow(
+    workflowExecution = await executeWorkflow(
       config.stackName,
       config.bucket,
       taskName,
@@ -40,7 +41,7 @@ describe("The Ingest Granules workflow", function() {
     let lambdaOutput = null;
 
     beforeAll(async function() {
-      lambdaOutput = await workflow.getLambdaOutput(workflowExecution.executionArn, "SyncGranule");
+      lambdaOutput = await lambdaStep.getStepOutput(workflowExecution.executionArn, "SyncGranule");
     });
 
     it("has expected payload", function() {

spec/parsePdr/ParsePdr.input.template.json

@@ -12,6 +12,8 @@
     },
     "collection": {
       "provider_path": "cumulus-test-data/pdrs",
+      "name": "test-collection",
+      "granuleIdExtraction": "(.*)",
       "files": []
     },
     "provider": {
@@ -37,9 +39,7 @@
       "provider": "{{$.meta.provider}}",
       "collection": "{{$.meta.collection}}",
       "granuleIngestMessageTemplateUri": "{{$.meta.templates.IngestGranule}}",
-      "queueUrl": "{{$.meta.queues.startSF}}",
-      "bucket": "{{$.meta.buckets.internal}}",
-      "stack": "{{$.meta.stack}}"
+      "queueUrl": "{{$.meta.queues.startSF}}"
     },
     "CheckStatus": {},
     "CheckAgainChoice": {},

spec/parsePdr/ParsePdr.output.json

@@ -10,6 +10,7 @@
   },
   "granules": [
     {
+      "granuleId": "MOD09GQ.A2016358.h13v04.006.2016360104606.hdf",
       "dataType": "MOD09GQ",
       "files": [
         {

spec/parsePdr/ParsePdrSuccessSpec.js

@@ -1,11 +1,13 @@
 const fs = require('fs');
 const { S3 } = require('aws-sdk');
-const workflow = require('@cumulus/integration-tests');
+const { executeWorkflow, LambdaStep } = require('@cumulus/integration-tests');
 
 const { loadConfig, templateFile } = require('../helpers/testUtils');
 
 const s3 = new S3();
 const config = loadConfig();
+const lambdaStep = new LambdaStep();
+
 const taskName = 'ParsePdr';
 const inputTemplateFilename = './spec/parsePdr/ParsePdr.input.template.json';
 const templatedInputFilename = templateFile({
@@ -21,7 +23,7 @@ describe("The Parse PDR workflow", function() {
   let workflowExecution = null;
 
   beforeAll(async function() {
-    workflowExecution = await workflow.executeWorkflow(
+    workflowExecution = await executeWorkflow(
       config.stackName,
       config.bucket,
       taskName,
@@ -44,7 +46,7 @@ describe("The Parse PDR workflow", function() {
     let lambdaOutput = null;
 
     beforeAll(async function() {
-      lambdaOutput = await workflow.getLambdaOutput(workflowExecution.executionArn, "ParsePdr");
+      lambdaOutput = await lambdaStep.getStepOutput(workflowExecution.executionArn, "ParsePdr");
     });
 
     it("has expected path and name output", function() {
@@ -56,7 +58,7 @@ describe("The Parse PDR workflow", function() {
     let lambdaOutput = null;
 
     beforeAll(async function() {
-      lambdaOutput = await workflow.getLambdaOutput(workflowExecution.executionArn, "QueueGranules");
+      lambdaOutput = await lambdaStep.getStepOutput(workflowExecution.executionArn, "QueueGranules");
     });
 
     it("has expected path and name output", function() {