From 13d9be1efc8740710af7859876abc01d4661bec1 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 Mar 2018 13:59:04 -0400 Subject: [PATCH 1/6] Release version 1.2.0 (#265) **Summary:** Release 1.2.0 * Update packages to 1.2.0 * Update changelog Reviewers: @laurenfrederick --- .circleci/config.yml | 23 +-- .eslint-ratchet-high-water-mark | 1 + .eslintignore | 3 + .eslintrc.json | 8 +- .github/pull_request_template.md | 2 + .gitignore | 5 +- CHANGELOG.md | 22 ++- README.md | 47 +++++++ bin/eslint-ratchet | 32 +++++ cumulus/services/sfn-scheduler/package.json | 4 +- cumulus/services/sfn-throttler/package.json | 4 +- .../copy-idx-from-s3-to-efs/package.json | 8 +- .../delete-ingest-tracking-data/package.json | 6 +- cumulus/tasks/delete-pdr-ftp/package.json | 6 +- cumulus/tasks/delete-pdr-s3/package.json | 8 +- .../tasks/discover-cmr-granules/package.json | 8 +- cumulus/tasks/discover-granules/package.json | 12 +- .../tasks/discover-granules/tests/index.js | 3 +- .../tasks/discover-http-tiles/package.json | 4 +- cumulus/tasks/discover-pdr/package.json | 8 +- cumulus/tasks/discover-pdrs/package.json | 12 +- .../tasks/discover-s3-granules/package.json | 18 +-- .../tasks/download-activity-mock/package.json | 6 +- cumulus/tasks/filter-payload/package.json | 6 +- cumulus/tasks/generate-mrf/package.json | 4 +- cumulus/tasks/generate-pan/package.json | 6 +- .../tasks/generate-pdr-file-list/package.json | 10 +- cumulus/tasks/generate-pdrd/package.json | 6 +- cumulus/tasks/hello-world/package.json | 4 +- cumulus/tasks/parse-pdr/package.json | 12 +- cumulus/tasks/pdr-status-check/index.js | 48 ++++--- cumulus/tasks/pdr-status-check/package.json | 12 +- .../tasks/pdr-status-check/schemas/input.json | 15 +- .../pdr-status-check/schemas/output.json | 15 +- cumulus/tasks/pdr-status-check/tests/index.js | 34 +++-- cumulus/tasks/post-to-cmr/package.json | 14 +- cumulus/tasks/queue-granules/package.json | 10 +- cumulus/tasks/queue-granules/tests/index.js | 3 +- cumulus/tasks/queue-pdrs/package.json | 11 +- cumulus/tasks/queue-pdrs/tests/index.js | 3 +- cumulus/tasks/run-gdal/package.json | 6 +- cumulus/tasks/sf-sns-report/README.md | 52 +++++++ cumulus/tasks/sf-sns-report/index.js | 131 ++++++++++++++++++ cumulus/tasks/sf-sns-report/package.json | 56 ++++++++ .../tasks/sf-sns-report/schemas/config.json | 36 +++++ .../tasks/sf-sns-report/tests/.eslintrc.json | 5 + cumulus/tasks/sf-sns-report/tests/index.js | 102 ++++++++++++++ cumulus/tasks/sf-sns-report/webpack.config.js | 19 +++ cumulus/tasks/sync-granule/package.json | 12 +- cumulus/tasks/sync-http-urls/package.json | 4 +- cumulus/tasks/sync-wms/package.json | 4 +- cumulus/tasks/tee/package.json | 4 +- cumulus/tasks/trigger-ingest/package.json | 4 +- cumulus/tasks/trigger-mrf-gen/package.json | 6 +- .../tasks/trigger-process-pdrs/package.json | 6 +- cumulus/tasks/validate-archives/package.json | 6 +- cumulus/tasks/validate-pdr/package.json | 16 +-- lerna.json | 2 +- package.json | 6 +- packages/api/config/lambdas.yml | 7 +- packages/api/endpoints/collections.js | 3 +- packages/api/endpoints/providers.js | 3 +- packages/api/endpoints/rules.js | 3 +- packages/api/es/search.js | 3 +- packages/api/lambdas/bootstrap.js | 17 ++- packages/api/lambdas/db-indexer.js | 13 +- packages/api/lambdas/jobs.js | 23 +-- packages/api/lambdas/kinesis-consumer.js | 25 +++- packages/api/lambdas/sf-scheduler.js | 13 +- packages/api/package.json | 21 +-- packages/api/tests/test-db-indexer.js | 1 - packages/api/tests/test-kinesis-consumer.js | 130 +++++++++-------- packages/cmrjs/package.json | 4 +- packages/common/aws.js | 52 +++++-- packages/common/package.json | 12 +- packages/common/test-utils.js | 32 +---- packages/ingest/README.md | 10 ++ packages/ingest/aws.js | 9 +- packages/ingest/crypto.js | 16 +-- packages/ingest/package.json | 12 +- packages/ingest/parse-pdr.js | 9 +- packages/ingest/pdr.js | 4 +- packages/ingest/queue.js | 29 +--- packages/ingest/test/queue.js | 53 ++++--- packages/integration-tests/package.json | 4 +- packages/pvl/package.json | 4 +- packages/task-debug/package.json | 6 +- .../cumulus_messages/pdr-status-check.json | 6 +- packages/test-data/package.json | 2 +- tests/ftp_pdr_parse_ingest.js | 3 +- tests/sftp_pdr_parse_ingest.js | 4 +- 91 files changed, 1038 insertions(+), 435 deletions(-) create mode 100644 .eslint-ratchet-high-water-mark create mode 100644 .eslintignore create mode 100755 bin/eslint-ratchet create mode 100644 cumulus/tasks/sf-sns-report/README.md create mode 100644 cumulus/tasks/sf-sns-report/index.js create mode 100644 cumulus/tasks/sf-sns-report/package.json create mode 100644 cumulus/tasks/sf-sns-report/schemas/config.json create mode 100644 cumulus/tasks/sf-sns-report/tests/.eslintrc.json create mode 100644 cumulus/tasks/sf-sns-report/tests/index.js create mode 100644 cumulus/tasks/sf-sns-report/webpack.config.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c62bd89125..130eddf0dac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,12 +16,12 @@ jobs: - core- - run: - name: core installation + name: core installation command: | yarn install # generate global hash - ./node_modules/.bin/lerna exec --concurrency 1 -- sha1sum package.json | awk '{print $1}' '' >> /home/circleci/project/global-hash + ./node_modules/.bin/lerna exec --concurrency 1 -- sha1sum package.json | awk '{print $1}' '' >> /home/circleci/project/global-hash - save_cache: paths: @@ -89,6 +89,7 @@ jobs: - ./cumulus/tasks/queue-pdrs/node_modules - ./cumulus/tasks/sync-granule/node_modules - ./cumulus/tasks/discover-s3-granules/node_modules + - ./cumulus/tasks/sf-sns-report/node_modules - save_cache: key: gitc-tasks-{{ checksum "global-hash" }} @@ -115,13 +116,13 @@ jobs: - ./cumulus/tasks/discover-pdr/node_modules - ./cumulus/tasks/generate-pdr-file-list/node_modules - ./cumulus/tasks/validate-pdr/node_modules - + - save_cache: key: services-{{ checksum "global-hash" }} paths: - ./cumulus/services/sfn-scheduler/node_modules - ./cumulus/services/sfn-throttler/node_modules - + - run: name: Running Tests @@ -134,7 +135,7 @@ jobs: environment: LOCALSTACK_HOST: localstack command: yarn e2e - + - run: name: Running AWS Integration Tests command: | @@ -152,8 +153,8 @@ jobs: cd ~/project # run the build command on the packages used on the integration tests - cat ../integration/cumulus_integration_tests_packages.txt | xargs -I % ./node_modules/.bin/lerna exec --scope % -- yarn build - + cat ../integration/cumulus_integration_tests_packages.txt | xargs -I % ./node_modules/.bin/lerna exec --scope % -- yarn build + # run the yarn link on packages used on the integration tests cat ../integration/cumulus_integration_tests_packages.txt | xargs -I % ./node_modules/.bin/lerna exec --scope % -- yarn link cd ~/integration @@ -167,7 +168,11 @@ jobs: # run the tests yarn test fi - + + - run: + name: Running eslint-ratchet + command: ./bin/eslint-ratchet ci + build_and_publish: docker: - image: circleci/node:6.10 @@ -196,4 +201,4 @@ workflows: - build_and_test filters: branches: - only: release \ No newline at end of file + only: release diff --git a/.eslint-ratchet-high-water-mark b/.eslint-ratchet-high-water-mark new file mode 100644 index 00000000000..988fde4112c --- /dev/null +++ b/.eslint-ratchet-high-water-mark @@ -0,0 +1 @@ +1873 diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000000..f8d56b13bdf --- /dev/null +++ b/.eslintignore @@ -0,0 +1,3 @@ +**/node_modules/** +**/dist/** +packages/deployment/app/*.js diff --git a/.eslintrc.json b/.eslintrc.json index 8cb2fb5368f..a401ce8cf0d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -78,8 +78,12 @@ "guard-for-in": "off", "object-shorthand": "off", "space-before-function-paren": [ - "warn", - "never" + "error", + { + "anonymous": "always", + "named": "never", + "asyncArrow": "always" + } ], "brace-style": [ 2, diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d3f21a3919a..4f4c19464ba 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,6 +13,8 @@ Things that should succeed before merging. - [ ] Unit tests - [ ] Adhoc testing - [ ] Update CHANGELOG +- [ ] Run `./bin/eslint-ratchet` and verify that eslint errors have not increased + - [ ] Commit `.eslint-ratchet-high-water-mark` if the score has improved ## Release PR diff --git a/.gitignore b/.gitignore index 5882eb1a119..1491bcfcd5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ package-lock.json yarn.lock +yarn-error.log node_modules build/ dist/ @@ -13,6 +14,6 @@ packaged-* .vscode *-debug.log profiles.clj -*.map +*.map .DS_Store -packages/deployment/app/*.js \ No newline at end of file +packages/deployment/app/*.js diff --git a/CHANGELOG.md b/CHANGELOG.md index d0204128e1b..f55117ca267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v1.2.0] - 2018-03-20 + +### Fixed +- Update vulnerable npm packages [CUMULUS-425] +- `@cumulus/api`: `kinesis-consumer.js` uses `sf-scheduler.js#schedule` instead of placing a message directly on the `startSF` SQS queue. This is a fix for [CUMULUS-359](https://bugs.earthdata.nasa.gov/browse/CUMULUS-359) because `sf-scheduler.js#schedule` looks up the provider and collection data in DynamoDB and adds it to the `meta` object of the enqueued message payload. +- `@cumulus/api`: `kinesis-consumer.js` catches and logs errors instead of doing an error callback. Before this change, `kinesis-consumer` was failing to process new records when an existing record caused an error because it would call back with an error and stop processing additional records. It keeps trying to process the record causing the error because it's "position" in the stream is unchanged. Catching and logging the errors is part 1 of the fix. Proposed part 2 is to enqueue the error and the message on a "dead-letter" queue so it can be processed later ([CUMULUS-413](https://bugs.earthdata.nasa.gov/browse/CUMULUS-413)). +- **CUMULUS-260: "PDR page on dashboard only shows zeros."** The PDR stats in LPDAAC are all 0s, even if the dashboard has been fixed to retrieve the correct fields. The current version of pdr-status-check has a few issues. + - pdr is not included in the input/output schema. It's available from the input event. So the pdr status and stats are not updated when the ParsePdr workflow is complete. Adding the pdr to the input/output of the task will fix this. + - pdr-status-check doesn't update pdr stats which prevent the real time pdr progress from showing up in the dashboard. To solve this, added lambda function sf-sns-report which is copied from @cumulus/api/lambdas/sf-sns-broadcast with modification, sf-sns-report can be used to report step function status anywhere inside a step function. So add step sf-sns-report after each pdr-status-check, we will get the PDR status progress at real time. + - It's possible an execution is still in the queue and doesn't exist in sfn yet. Added code to handle 'ExecutionDoesNotExist' error when checking the execution status. +- Fixed `aws.cloudwatchevents()` typo in `packages/ingest/aws.js`. This typo was the root cause of the error: `Error: Could not process scheduled_ingest, Error: : aws.cloudwatchevents is not a constructor` seen when trying to update a rule. + +### Removed + +- `@cumulus/ingest/aws`: Remove queueWorkflowMessage which is no longer being used by `@cumulus/api`'s `kinesis-consumer.js`. + ## [v1.1.4] - 2018-03-15 + ### Added - added flag `useList` to parse-pdr [CUMULUS-404] ### Fixed + - Pass encrypted password to the ApiGranule Lambda function [CUMULUS-424] + ## [v1.1.3] - 2018-03-14 ### Fixed - Changed @cumulus/deployment package install behavior. The build process will happen after installation @@ -79,7 +98,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v1.0.0] - 2018-02-23 -[Unreleased]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.4...HEAD +[Unreleased]: https://github.com/cumulus-nasa/cumulus/compare/v1.2.0...HEAD +[v1.2.0]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.4...v1.2.0 [v1.1.4]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.3...v1.1.4 [v1.1.3]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.2...v1.1.3 [v1.1.2]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.1...v1.1.2 diff --git a/README.md b/README.md index 5e24e96d5a9..1aee8f3ee97 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,53 @@ Run end to end tests by $ yarn e2e +## Code quality checking + +This project uses [eslint](https://eslint.org/) to check code style and quality. +The configured eslint rules can be found in the project's +[.eslintrc.json](https://github.com/cumulus-nasa/cumulus/blob/master/.eslintrc.json) +file. + +In an effort to gradually reduce the number of eslint errors in our codebase, +we are using a script called `eslint-ratchet`. It runs `eslint` against the +repo and compares the number of errors to the previous number of errors. The +previous number of errors is stored in the `.eslint-ratchet-high-water-mark` +file, and tracked in git. If the script is run and the number of errors has +been reduced, the new, lower score is stored in +`.eslint-ratchet-high-water-mark` and should be committed into git. If the +number of errors has increased, the script will fail and tell you that the +number of errors has increased. + +To run the script, simply run `./bin/eslint-ratchet` from the top of the +cumulus repository. + +The `eslint-ratchet` script is also part of our CircleCI build. If the number +of eslint errors that CircleCI finds has increased, it will fail the build. If +the number of errors has *decreased* from what is stored in +`.eslint-ratchet-high-water-mark`, it will also fail the build. In that case, +run `./bin/eslint-ratchet` and commit the new-and-improved +`.eslint-ratchet-high-water-mark` file. + +To help prevent unexpected build failures in CircleCI, I suggest adding a +local post-commit hook that will run eslint-ratchet after every commit. This +will not cause your commits to fail if the score has increased, but it will +let you know that there is a problem. To set up the post-commit hook, create a +file called `.git/hooks/post-commit` which contains: + +``` +#!/bin/sh + +set -e + +echo "Running ./bin/eslint-ratchet" +./bin/eslint-ratchet +``` + +Make sure the hook is executable with `chmod +x .git/hooks/post-commit` + +This idea of ratcheting down the number of errors came from Vince Broz's +excellent [quality](https://github.com/apiology/quality) gem. + ## Adding New Packages Create a new folder under `packages` if it is a common library or create folder under `cumulus/tasks` if it is a lambda task. `cd` to the folder and run `npm init`. diff --git a/bin/eslint-ratchet b/bin/eslint-ratchet new file mode 100755 index 00000000000..6d9893618b5 --- /dev/null +++ b/bin/eslint-ratchet @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Only valid option is "ci" +ARG="$1" +if [ "$ARG" != "" ] && [ "$ARG" != "ci" ]; then + echo "ERROR: Invalid argument - ${ARG}" >&2 + exit 1; +fi + +PREV=$(< .eslint-ratchet-high-water-mark) +NOW=$(./node_modules/.bin/eslint --format unix . | tail -n 1 | awk '{ print $1 }') + +# If there are more errors now than there used to be, fail +if (( NOW > PREV )); then + DELTA=$(( NOW - PREV)) + echo "eslint-ratchet error: eslint errors have increased from ${PREV} to ${NOW}" >&2 + exit 1 +elif (( NOW < PREV )); then + if [ "$ARG" == "ci" ]; then + echo "eslint-ratchet error: Expected ${PREV} errors but found ${NOW}. Run './bin/eslint-ratchet' and commit .eslint-ratchet-high-water-mark file" >&2 + exit 2 + else + echo "WIN: ESLint errors have decreased from ${PREV} to ${NOW}. Ratcheting down ..." + echo " Make sure to commit .eslint-ratchet-high-water-mark" + echo "$NOW" > .eslint-ratchet-high-water-mark + exit 0 + fi +else + exit 0 +fi diff --git a/cumulus/services/sfn-scheduler/package.json b/cumulus/services/sfn-scheduler/package.json index 1d0e5376aea..82007cc4fff 100644 --- a/cumulus/services/sfn-scheduler/package.json +++ b/cumulus/services/sfn-scheduler/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/sfn-scheduler", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Runs ingest timers as configured in /config/products.json", "main": "index.js", "keywords": [ @@ -17,7 +17,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "lodash": "^4.17.4", "uuid": "^3.0.1" } diff --git a/cumulus/services/sfn-throttler/package.json b/cumulus/services/sfn-throttler/package.json index 081d9886e82..39e591fbeb2 100644 --- a/cumulus/services/sfn-throttler/package.json +++ b/cumulus/services/sfn-throttler/package.json @@ -1,6 +1,6 @@ { "name": "aws-step-function-throttling-service", - "version": "1.0.0", + "version": "1.2.0", "private": true, "description": "A service to throttle concurrent executions of AWS Step Functions.", "main": "index.js", @@ -18,7 +18,7 @@ "lodash": "^4.17.4" }, "devDependencies": { - "eslint": "^4.10.0", + "eslint": "^4.18.2", "eslint-config-airbnb-base": "^12.1.0", "eslint-plugin-import": "^2.8.0" } diff --git a/cumulus/tasks/copy-idx-from-s3-to-efs/package.json b/cumulus/tasks/copy-idx-from-s3-to-efs/package.json index d38e610d533..b092c5064bc 100644 --- a/cumulus/tasks/copy-idx-from-s3-to-efs/package.json +++ b/cumulus/tasks/copy-idx-from-s3-to-efs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/copy-idx-from-s3-to-efs", - "version": "1.1.3", + "version": "1.2.0", "description": "Task to copy idx file from S3 to EFS for OnEarth 2.0.", "main": "index.js", "private": true, @@ -35,13 +35,13 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", - "@cumulus/test-data": "^1.1.3" + "@cumulus/common": "^1.2.0", + "@cumulus/test-data": "^1.2.0" }, "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", diff --git a/cumulus/tasks/delete-ingest-tracking-data/package.json b/cumulus/tasks/delete-ingest-tracking-data/package.json index fb48d737f05..dd40e83a066 100644 --- a/cumulus/tasks/delete-ingest-tracking-data/package.json +++ b/cumulus/tasks/delete-ingest-tracking-data/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/delete-ingest-tracking-data", - "version": "1.1.3", + "version": "1.2.0", "description": "Task to delete ingest tracking data when ingest has completed.", "main": "index.js", "private": true, @@ -36,12 +36,12 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3" + "@cumulus/common": "^1.2.0" }, "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/delete-pdr-ftp/package.json b/cumulus/tasks/delete-pdr-ftp/package.json index b8d2b8affac..17f6e016b78 100644 --- a/cumulus/tasks/delete-pdr-ftp/package.json +++ b/cumulus/tasks/delete-pdr-ftp/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/delete-pdr-ftp", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "SIPS handler PDR discovery task", "main": "index.js", "keywords": [ @@ -35,7 +35,7 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -47,7 +47,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/delete-pdr-s3/package.json b/cumulus/tasks/delete-pdr-s3/package.json index f18da8212e7..efb80178d89 100644 --- a/cumulus/tasks/delete-pdr-s3/package.json +++ b/cumulus/tasks/delete-pdr-s3/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/delete-pdr-s3", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "SIPS handler PDR discovery task", "main": "index.js", "keywords": [ @@ -18,7 +18,7 @@ "test": "test" }, "scripts": { - "test": "env TEST=true ava" + "test": "ava" }, "ava": { "files": [ @@ -39,7 +39,7 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -51,7 +51,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/discover-cmr-granules/package.json b/cumulus/tasks/discover-cmr-granules/package.json index 425cdd803cc..0f12ac9fa41 100644 --- a/cumulus/tasks/discover-cmr-granules/package.json +++ b/cumulus/tasks/discover-cmr-granules/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/discover-cmr-granules", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Discovers granules from the CMR", "main": "index.js", "keywords": [ @@ -19,8 +19,8 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", - "moment": "^2.19.2", + "@cumulus/common": "^1.2.0", + "moment": "^2.21.0", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1" }, @@ -39,7 +39,7 @@ ] }, "devDependencies": { - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/discover-granules/package.json b/cumulus/tasks/discover-granules/package.json index 435fd708cac..b9583a1a641 100644 --- a/cumulus/tasks/discover-granules/package.json +++ b/cumulus/tasks/discover-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-granules", - "version": "1.1.4", + "version": "1.2.0", "description": "Discover Granules in FTP/HTTP/SFTP endpoints", "main": "index.js", "directories": { @@ -11,7 +11,7 @@ "url": "https://github.com/cumulus-nasa/cumulus" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "local": "node index.js | pino", "build": "webpack --progress", "watch": "webpack --progress -w", @@ -37,10 +37,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", @@ -52,7 +52,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "fs-extra": "^5.0.0", "lodash.clonedeep": "^4.5.0" } diff --git a/cumulus/tasks/discover-granules/tests/index.js b/cumulus/tasks/discover-granules/tests/index.js index fd6f16100ab..7689048a1c4 100644 --- a/cumulus/tasks/discover-granules/tests/index.js +++ b/cumulus/tasks/discover-granules/tests/index.js @@ -14,7 +14,8 @@ const { } = require('@cumulus/common/test-utils'); const { discoverGranules } = require('../index'); -test('discover granules using FTP', async (t) => { +// This test is broken and will be fixed by CUMULUS-427 +test.skip('discover granules using FTP', async (t) => { const event = cloneDeep(mur); await validateConfig(t, event.config); diff --git a/cumulus/tasks/discover-http-tiles/package.json b/cumulus/tasks/discover-http-tiles/package.json index 4b61b9b1f67..6131538c0e7 100644 --- a/cumulus/tasks/discover-http-tiles/package.json +++ b/cumulus/tasks/discover-http-tiles/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/discover-http-tiles", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Crawls an HTTP endpoint to discover tiled imagery", "main": "index.js", "keywords": [ @@ -14,7 +14,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "async": "^2.0.0", "lodash": "^4.15.0", "simplecrawler": "git+https://github.com/cgiffard/node-simplecrawler.git#193e506c39164ddf045c7c6c502e1a015d85a290" diff --git a/cumulus/tasks/discover-pdr/package.json b/cumulus/tasks/discover-pdr/package.json index a49ae6aaf79..9c0df38fd5f 100644 --- a/cumulus/tasks/discover-pdr/package.json +++ b/cumulus/tasks/discover-pdr/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/discover-pdr", "private": true, - "version": "1.1.4", + "version": "1.2.0", "description": "SIPS handler PDR discovery task", "main": "index.js", "keywords": [ @@ -36,8 +36,8 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.4", + "@cumulus/common": "^1.2.0", + "@cumulus/ingest": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -50,7 +50,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/discover-pdrs/package.json b/cumulus/tasks/discover-pdrs/package.json index 4d493ea8c03..2a5f41da51a 100644 --- a/cumulus/tasks/discover-pdrs/package.json +++ b/cumulus/tasks/discover-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-pdrs", - "version": "1.1.4", + "version": "1.2.0", "description": "Discover PDRs in FTP and HTTP endpoints", "main": "index.js", "directories": { @@ -11,7 +11,7 @@ "url": "https://github.com/cumulus-nasa/cumulus" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "build": "mkdir -p dist && cp cumulus.json dist/ && webpack --progress", "watch": "mkdir -p dist && cp cumulus.json dist/ && webpack --progress -w", "postinstall": "npm run build" @@ -36,10 +36,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", @@ -51,7 +51,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "fs-extra": "^5.0.0", "lodash": "^4.17.5" } diff --git a/cumulus/tasks/discover-s3-granules/package.json b/cumulus/tasks/discover-s3-granules/package.json index a70618d4523..1061fe29fc5 100644 --- a/cumulus/tasks/discover-s3-granules/package.json +++ b/cumulus/tasks/discover-s3-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-s3-granules", - "version": "1.1.4", + "version": "1.2.0", "description": "Discover granules from an S3 bucket", "main": "index.js", "directories": { @@ -14,18 +14,20 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava tests/*.js --serial", + "test": "ava", "local": "node index.js | pino", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" }, "ava": { + "files": "tests", "babel": "inherit", "require": [ "babel-polyfill", "babel-register" - ] + ], + "serial": true }, "babel": { "presets": [ @@ -35,22 +37,22 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", "babel-preset-es2017": "^6.24.1", "json-loader": "~0.5.7", "lodash.get": "^4.4.2", - "moment": "2.18.1", + "moment": "^2.21.0", "webpack": "~1.12.13" }, "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0" + "ava": "^0.23.0" } } diff --git a/cumulus/tasks/download-activity-mock/package.json b/cumulus/tasks/download-activity-mock/package.json index 1552ed00bbf..d4bca34200a 100644 --- a/cumulus/tasks/download-activity-mock/package.json +++ b/cumulus/tasks/download-activity-mock/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/dowload-activity-mock", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Mock for provider gateway", "main": "index.js", "keywords": [ @@ -35,7 +35,7 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -47,7 +47,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/filter-payload/package.json b/cumulus/tasks/filter-payload/package.json index d840b4dfb3e..36ebf075d8b 100644 --- a/cumulus/tasks/filter-payload/package.json +++ b/cumulus/tasks/filter-payload/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/filter-payload", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Task to filter the payload from one task to the next", "main": "index.js", "keywords": [ @@ -35,12 +35,12 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3" + "@cumulus/common": "^1.2.0" }, "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/generate-mrf/package.json b/cumulus/tasks/generate-mrf/package.json index 3e0e1d53b86..c22d1756145 100644 --- a/cumulus/tasks/generate-mrf/package.json +++ b/cumulus/tasks/generate-mrf/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate-mrf", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Generates MRFs for ingested tiles", "main": "index.js", "keywords": [ @@ -14,7 +14,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "async": "^2.0.0", "lodash": "^4.13.1", "mustache": "^2.2.1", diff --git a/cumulus/tasks/generate-pan/package.json b/cumulus/tasks/generate-pan/package.json index c8ad6dd8331..4a6946725ff 100644 --- a/cumulus/tasks/generate-pan/package.json +++ b/cumulus/tasks/generate-pan/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate-pan", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "SIPS handler PAN generation/upload task", "main": "index.js", "keywords": [ @@ -35,7 +35,7 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -47,7 +47,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/generate-pdr-file-list/package.json b/cumulus/tasks/generate-pdr-file-list/package.json index 502adc19079..d4b93a0dd2f 100644 --- a/cumulus/tasks/generate-pdr-file-list/package.json +++ b/cumulus/tasks/generate-pdr-file-list/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate-pdr-file-list", "private": true, - "version": "1.1.4", + "version": "1.2.0", "description": "SIPS handler PDR processing task", "main": "index.js", "keywords": [ @@ -35,9 +35,9 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.4", - "@cumulus/pvl": "^1.0.0", + "@cumulus/common": "^1.2.0", + "@cumulus/ingest": "^1.2.0", + "@cumulus/pvl": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -49,7 +49,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/generate-pdrd/package.json b/cumulus/tasks/generate-pdrd/package.json index 1c5a4397ef6..5d18bfbc904 100644 --- a/cumulus/tasks/generate-pdrd/package.json +++ b/cumulus/tasks/generate-pdrd/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate-pdrd", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "SIPS handler generating a PDRD when PDR validation fails", "main": "index.js", "keywords": [ @@ -35,7 +35,7 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -47,7 +47,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/hello-world/package.json b/cumulus/tasks/hello-world/package.json index f8a1f309dbe..5c5efc30f58 100644 --- a/cumulus/tasks/hello-world/package.json +++ b/cumulus/tasks/hello-world/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/hello-world", - "version": "1.1.3", + "version": "1.2.0", "description": "Example task", "main": "index.js", "directories": { @@ -49,7 +49,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "proxyquire": "^1.8.0" } } diff --git a/cumulus/tasks/parse-pdr/package.json b/cumulus/tasks/parse-pdr/package.json index 368b45466a9..b9a282125b2 100644 --- a/cumulus/tasks/parse-pdr/package.json +++ b/cumulus/tasks/parse-pdr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/parse-pdr", - "version": "1.1.4", + "version": "1.2.0", "description": "Download and Parse a given PDR", "main": "index.js", "directories": { @@ -14,7 +14,7 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" @@ -39,10 +39,10 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", @@ -55,7 +55,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "fs-extra": "^5.0.0", "lodash.clonedeep": "^4.5.0", "proxyquire": "^1.8.0" diff --git a/cumulus/tasks/pdr-status-check/index.js b/cumulus/tasks/pdr-status-check/index.js index 75f06f25815..74847119843 100644 --- a/cumulus/tasks/pdr-status-check/index.js +++ b/cumulus/tasks/pdr-status-check/index.js @@ -78,7 +78,8 @@ function logStatus(output) { * failed: [ * { arn: 'arn:456', reason: 'Workflow Aborted' } * ], - * completed: [] + * completed: [], + * pdr: {} * } * * @param {Object} event - the event that came into checkPdrStatuses @@ -117,7 +118,8 @@ function buildOutput(event, groupedExecutions) { isFinished: groupedExecutions.running.length === 0, running, failed, - completed + completed, + pdr: event.input.pdr }; if (!output.isFinished) { @@ -137,28 +139,36 @@ function buildOutput(event, groupedExecutions) { * @returns {Promise.} - an object describing the status of Step * Function executions related to a PDR */ -function checkPdrStatuses(event) { +async function checkPdrStatuses(event) { const runningExecutionArns = event.input.running || []; - const promisedExecutionDescriptions = runningExecutionArns.map((executionArn) => - aws.sfn().describeExecution({ executionArn }).promise()); - - return Promise.all(promisedExecutionDescriptions) - .then(groupExecutionsByStatus) - .then((groupedExecutions) => { - const counter = getCounterFromEvent(event) + 1; - const exceededLimit = counter >= getLimitFromEvent(event); + const executions = []; + for (const executionArn of runningExecutionArns) { + try { + const execution = await aws.sfn().describeExecution({ executionArn }).promise(); + executions.push(execution); + } + catch (e) { + // it's ok if a execution is still in the queue and has not be executed + if (e.code === 'ExecutionDoesNotExist') { + executions.push({ executionArn: executionArn, status: 'RUNNING' }); + } + else throw e; + } + } - const executionsAllDone = groupedExecutions.running.length === 0; + const groupedExecutions = groupExecutionsByStatus(executions); + const counter = getCounterFromEvent(event) + 1; + const exceededLimit = counter >= getLimitFromEvent(event); - if (!executionsAllDone && exceededLimit) { - throw new IncompleteError(`PDR didn't complete after ${counter} checks`); - } + const executionsAllDone = groupedExecutions.running.length === 0; + if (!executionsAllDone && exceededLimit) { + throw new IncompleteError(`PDR didn't complete after ${counter} checks`); + } - const output = buildOutput(event, groupedExecutions); - if (!output.isFinished) logStatus(output); - return output; - }); + const output = buildOutput(event, groupedExecutions); + if (!output.isFinished) logStatus(output); + return output; } exports.checkPdrStatuses = checkPdrStatuses; diff --git a/cumulus/tasks/pdr-status-check/package.json b/cumulus/tasks/pdr-status-check/package.json index 25ef23a59b9..bace424943f 100644 --- a/cumulus/tasks/pdr-status-check/package.json +++ b/cumulus/tasks/pdr-status-check/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pdr-status-check", - "version": "1.1.4", + "version": "1.2.0", "description": "Checks execution status of granules in a PDR", "main": "index.js", "directories": { @@ -14,7 +14,7 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" @@ -40,10 +40,10 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", @@ -57,7 +57,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "lodash": "^4.17.5", "proxyquire": "^1.8.0", "sinon": "^2.0.0-pre.5" diff --git a/cumulus/tasks/pdr-status-check/schemas/input.json b/cumulus/tasks/pdr-status-check/schemas/input.json index 82fe1fe8e52..7c8d82d1ba9 100644 --- a/cumulus/tasks/pdr-status-check/schemas/input.json +++ b/cumulus/tasks/pdr-status-check/schemas/input.json @@ -2,6 +2,7 @@ "title": "PdrStatusCheckInput", "description": "Describes the input expected by the pdr-status-check task", "type": "object", + "required": ["running", "pdr"], "properties": { "running": { "type": "array", @@ -23,6 +24,18 @@ }, "counter": { "type": "integer" }, "limit": { "type": "integer" }, - "isFinished": { "type": "boolean" } + "isFinished": { + "description": "Indicates whether all the step function executions of the PDR are in terminal states", + "type": "boolean" + }, + "pdr": { + "description": "Product Delivery Record", + "type": "object", + "required": ["name", "path"], + "properties": { + "name": { "type": "string" }, + "path": { "type": "string" } + } + } } } diff --git a/cumulus/tasks/pdr-status-check/schemas/output.json b/cumulus/tasks/pdr-status-check/schemas/output.json index 7bcc74594a3..0a9d4bfd381 100644 --- a/cumulus/tasks/pdr-status-check/schemas/output.json +++ b/cumulus/tasks/pdr-status-check/schemas/output.json @@ -2,6 +2,7 @@ "title": "PdrStatusCheckOutput", "description": "Describes the output produced by the pdr-status-check task", "type": "object", + "required": ["running", "completed", "failed", "isFinished", "pdr"], "properties": { "running": { "type": "array", @@ -23,6 +24,18 @@ }, "counter": { "type": "integer" }, "limit": { "type": "integer" }, - "isFinished": { "type": "boolean" } + "isFinished": { + "description": "Indicates whether all the step function executions of the PDR are in terminal states", + "type": "boolean" + }, + "pdr": { + "description": "Product Delivery Record", + "type": "object", + "required": ["name", "path"], + "properties": { + "name": { "type": "string" }, + "path": { "type": "string" } + } + } } } diff --git a/cumulus/tasks/pdr-status-check/tests/index.js b/cumulus/tasks/pdr-status-check/tests/index.js index 9b3d5b11fc6..4c8edc8f416 100644 --- a/cumulus/tasks/pdr-status-check/tests/index.js +++ b/cumulus/tasks/pdr-status-check/tests/index.js @@ -9,7 +9,8 @@ const { checkPdrStatuses } = require('../index'); test('valid output when no running executions', (t) => { const event = { input: { - running: [] + running: [], + pdr: { name: 'test.PDR', path: 'test-path' } } }; @@ -19,7 +20,8 @@ test('valid output when no running executions', (t) => { isFinished: true, running: [], failed: [], - completed: [] + completed: [], + pdr: { name: 'test.PDR', path: 'test-path' } }; t.deepEqual(output, expectedOutput); @@ -41,7 +43,8 @@ test('error thrown when limit exceeded', (t) => { input: { running: ['arn:123'], counter: 2, - limit: 3 + limit: 3, + pdr: { name: 'test.PDR', path: 'test-path' } } }; @@ -61,26 +64,35 @@ test('returns the correct results in the nominal case', (t) => { 'arn:1': 'RUNNING', 'arn:2': 'SUCCEEDED', 'arn:3': 'FAILED', - 'arn:4': 'ABORTED' + 'arn:4': 'ABORTED', + 'arn:7': null }; const stubSfnClient = { describeExecution: ({ executionArn }) => ({ - promise: () => Promise.resolve({ - executionArn, - status: executionStatuses[executionArn] - }) + promise: () => { + if (!executionStatuses[executionArn]) { + const error = new Error(`Execution does not exist: ${executionArn}`); + error.code = 'ExecutionDoesNotExist'; + return Promise.reject(error); + } + return Promise.resolve({ + executionArn, + status: executionStatuses[executionArn] + }); + } }) }; const stub = sinon.stub(aws, 'sfn').returns(stubSfnClient); const event = { input: { - running: ['arn:1', 'arn:2', 'arn:3', 'arn:4'], + running: ['arn:1', 'arn:2', 'arn:3', 'arn:4', 'arn:7'], completed: ['arn:5'], failed: [{ arn: 'arn:6', reason: 'OutOfCheese' }], counter: 5, - limit: 10 + limit: 10, + pdr: { name: 'test.PDR', path: 'test-path' } } }; @@ -92,7 +104,7 @@ test('returns the correct results in the nominal case', (t) => { t.is(output.counter, 6); t.is(output.limit, 10); - t.deepEqual(output.running, ['arn:1']); + t.deepEqual(output.running, ['arn:1', 'arn:7']); t.deepEqual(output.completed.sort(), ['arn:2', 'arn:5'].sort()); t.is(output.failed.length, 3); diff --git a/cumulus/tasks/post-to-cmr/package.json b/cumulus/tasks/post-to-cmr/package.json index a0cbffabf0f..311975ed78c 100644 --- a/cumulus/tasks/post-to-cmr/package.json +++ b/cumulus/tasks/post-to-cmr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/post-to-cmr", - "version": "1.1.4", + "version": "1.2.0", "description": "Post a given granule to CMR", "main": "index.js", "directories": { @@ -14,7 +14,7 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" @@ -41,11 +41,11 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/cmrjs": "^1.1.3", - "@cumulus/common": "^1.1.3", + "@cumulus/cmrjs": "^1.2.0", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", @@ -59,7 +59,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "proxyquire": "^1.8.0", "sinon": "^2.0.0-pre.5" } diff --git a/cumulus/tasks/queue-granules/package.json b/cumulus/tasks/queue-granules/package.json index 497993a3507..a25280af232 100644 --- a/cumulus/tasks/queue-granules/package.json +++ b/cumulus/tasks/queue-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-granules", - "version": "1.1.4", + "version": "1.2.0", "description": "Add discovered granules to the queue", "main": "index.js", "directories": { @@ -14,7 +14,7 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" @@ -38,9 +38,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", + "@cumulus/ingest": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", @@ -52,7 +52,7 @@ }, "devDependencies": { "@mapbox/mock-aws-sdk-js": "0.0.5", - "ava": "^0.21.0", + "ava": "^0.23.0", "lodash": "^4.17.5" } } diff --git a/cumulus/tasks/queue-granules/tests/index.js b/cumulus/tasks/queue-granules/tests/index.js index 8ed631aa4d7..774f04d3081 100644 --- a/cumulus/tasks/queue-granules/tests/index.js +++ b/cumulus/tasks/queue-granules/tests/index.js @@ -2,9 +2,8 @@ const test = require('ava'); -const { s3, sqs, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); +const { createQueue, s3, sqs, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); const { - createQueue, randomString, validateConfig, validateInput, diff --git a/cumulus/tasks/queue-pdrs/package.json b/cumulus/tasks/queue-pdrs/package.json index f3f478f5cd3..8a942046557 100644 --- a/cumulus/tasks/queue-pdrs/package.json +++ b/cumulus/tasks/queue-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-pdrs", - "version": "1.1.4", + "version": "1.2.0", "description": "Add discovered PDRs to a queue", "main": "index.js", "directories": { @@ -14,12 +14,13 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava tests/*.js", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" }, "ava": { + "files": "tests", "babel": "inherit", "require": [ "babel-polyfill", @@ -37,9 +38,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", + "@cumulus/ingest": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", @@ -51,6 +52,6 @@ }, "devDependencies": { "@mapbox/mock-aws-sdk-js": "0.0.5", - "ava": "^0.21.0" + "ava": "^0.23.0" } } diff --git a/cumulus/tasks/queue-pdrs/tests/index.js b/cumulus/tasks/queue-pdrs/tests/index.js index 6f94fa69b35..99e0f022061 100644 --- a/cumulus/tasks/queue-pdrs/tests/index.js +++ b/cumulus/tasks/queue-pdrs/tests/index.js @@ -2,9 +2,8 @@ const test = require('ava'); -const { s3, sqs, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); +const { createQueue, s3, sqs, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); const { - createQueue, randomString, validateConfig, validateInput, diff --git a/cumulus/tasks/run-gdal/package.json b/cumulus/tasks/run-gdal/package.json index a2dd4b1e7c2..f3d2f82f74d 100644 --- a/cumulus/tasks/run-gdal/package.json +++ b/cumulus/tasks/run-gdal/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "@cumulus/run-gdal", - "version": "1.1.3", + "version": "1.2.0", "description": "Runs gdal commands as a task", "main": "index.js", "keywords": [ @@ -20,7 +20,7 @@ "clean": "rm -rf dist" }, "devDependencies": { - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", @@ -28,6 +28,6 @@ "webpack": "^1.12.13" }, "dependencies": { - "@cumulus/common": "^1.1.3" + "@cumulus/common": "^1.2.0" } } diff --git a/cumulus/tasks/sf-sns-report/README.md b/cumulus/tasks/sf-sns-report/README.md new file mode 100644 index 00000000000..3461c4a83bf --- /dev/null +++ b/cumulus/tasks/sf-sns-report/README.md @@ -0,0 +1,52 @@ +# @cumulus/sf-sns-report + +[![CircleCI](https://circleci.com/gh/cumulus-nasa/cumulus.svg?style=svg)](https://circleci.com/gh/cumulus-nasa/cumulus) + +Broadcast an incoming Cumulus message to SNS. This lambda function works with Cumulus Message Adapter, and it can be used anywhere in a step function workflow to report granule and PDR status. + +To report the PDR's progress as it's being processed, add the following step after the pdr-status-check: + + PdrStatusReport: + CumulusConfig: + cumulus_message: + input: '{$}' + ResultPath: null + Type: Task + Resource: ${SfSnsReportLambdaFunction.Arn} + +To report the start status of the step function: + + StartAt: StatusReport + States: + StatusReport: + CumulusConfig: + cumulus_message: + input: '{$}' + ResultPath: null + Type: Task + Resource: ${SfSnsReportLambdaFunction.Arn} + +To report the final status of the step function: + + StopStatus: + CumulusConfig: + sfnEnd: true + stack: '{$.meta.stack}' + bucket: '{$.meta.buckets.internal}' + stateMachine: '{$.cumulus_meta.state_machine}' + executionName: '{$.cumulus_meta.execution_name}' + cumulus_message: + input: '{$}' + ResultPath: null + Type: Task + Resource: ${SfSnsReportLambdaFunction.Arn} + +## What is Cumulus? + +Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams. + +[Cumulus Documentation](https://cumulus-nasa.github.io/) + +## Contributing + +See [Cumulus README](https://github.com/cumulus-nasa/cumulus/blob/master/README.md#installing-and-deploying) diff --git a/cumulus/tasks/sf-sns-report/index.js b/cumulus/tasks/sf-sns-report/index.js new file mode 100644 index 00000000000..c30341e1102 --- /dev/null +++ b/cumulus/tasks/sf-sns-report/index.js @@ -0,0 +1,131 @@ +'use strict'; + +const get = require('lodash.get'); +const { setGranuleStatus, sns } = require('@cumulus/common/aws'); +const errors = require('@cumulus/common/errors'); +const cumulusMessageAdapter = require('@cumulus/cumulus-message-adapter-js'); + +/** + * Determines if there was a valid exception in the input message + * + * @param {Object} event - aws event object + * @returns {boolean} true if there was an exception, false otherwise + */ +function eventFailed(event) { + // event has exception + // and it is needed to avoid flagging cases like "exception: {}" or "exception: 'none'" + if ((event.exception) && (typeof event.exception === 'object') && + (Object.keys(event.exception).length > 0)) return true; + + // Error and error keys are not part of the cumulus message + // and if they appear in the message something is seriously wrong + else if (event.Error || event.error) return true; + + return false; +} + +/** + * Builds error object based on error type + * + * @param {string} type - error type + * @param {string} cause - error cause + * @returns {Object} the error object + */ +function buildError(type, cause) { + let ErrorClass; + + if (Object.keys(errors).includes(type)) ErrorClass = errors[type]; + else if (type === 'TypeError') ErrorClass = TypeError; + else ErrorClass = Error; + + return new ErrorClass(cause); +} + +/** + * If the cumulus message shows that a previous step failed, + * this function extracts the error message from the cumulus message + * and fails the function with that information. This ensures that the + * Step Function workflow fails with the correct error info + * + * @param {Object} event - aws event object + * @returns {undefined} throws an error and does not return anything + */ +function makeLambdaFunctionFail(event) { + const error = event.exception || event.error; + + if (error) throw buildError(error.Error, error.Cause); + + throw new Error('Step Function failed for an unknown reason.'); +} + +/** + * Publishes incoming Cumulus Message in its entirety to + * a given SNS topic + * + * @param {Object} event - a Cumulus Message that has been sent through the + * Cumulus Message Adapter. See schemas/input.json for detailed input schema. + * @param {Object} event.config - configuration object for the task + * @param {Object} event.config.sfnEnd - indicate if it's the last step of the step function + * @param {string} event.config.stack - the name of the deployment stack + * @param {string} event.config.bucket - S3 bucket + * @param {string} event.config.stateMachine - current state machine + * @param {string} event.config.executionName - execution name + * @returns {Promise.} - AWS SNS response or error in case of step function + * failure. + */ +async function publishSnsMessage(event) { + const config = get(event, 'config'); + const message = get(event, 'input'); + + const finished = get(config, 'sfnEnd', false); + const topicArn = get(message, 'meta.topic_arn', null); + const failed = eventFailed(message); + + let response = {}; + if (topicArn) { + // if this is the sns call at the end of the execution + if (finished) { + message.meta.status = failed ? 'failed' : 'completed'; + const granuleId = get(message, 'meta.granuleId', null); + if (granuleId) { + await setGranuleStatus( + granuleId, + config.stack, + config.bucket, + config.stateMachine, + config.executionName, + message.meta.status + ); + } + } + else { + message.meta.status = 'running'; + } + + response = await sns().publish({ + TopicArn: topicArn, + Message: JSON.stringify(message) + }).promise(); + } + + if (failed) { + makeLambdaFunctionFail(message); + } + + return response; +} + +exports.publishSnsMessage = publishSnsMessage; + +/** + * Lambda handler. It broadcasts an incoming Cumulus message to SNS + * + * @param {Object} event - a Cumulus Message + * @param {Object} context - an AWS Lambda context object + * @param {Function} callback - an AWS Lambda call back + * @returns {undefined} - does not return a value + */ +function handler(event, context, callback) { + cumulusMessageAdapter.runCumulusTask(publishSnsMessage, event, context, callback); +} +exports.handler = handler; diff --git a/cumulus/tasks/sf-sns-report/package.json b/cumulus/tasks/sf-sns-report/package.json new file mode 100644 index 00000000000..a7c6bdb2e09 --- /dev/null +++ b/cumulus/tasks/sf-sns-report/package.json @@ -0,0 +1,56 @@ +{ + "name": "@cumulus/sf-sns-report", + "version": "1.2.0", + "description": "Broadcasts an incoming Cumulus message to SNS", + "main": "index.js", + "directories": { + "test": "tests" + }, + "repository": { + "type": "git", + "url": "https://github.com/cumulus-nasa/cumulus" + }, + "publishConfig": { + "access": "public" + }, + "scripts": { + "test": "ava", + "build": "webpack --progress", + "watch": "webpack --progress -w", + "postinstall": "npm run build" + }, + "ava": { + "files": "tests", + "babel": "inherit", + "require": [ + "babel-polyfill", + "babel-register" + ] + }, + "babel": { + "presets": [ + "es2015" + ], + "plugins": [ + "transform-async-to-generator" + ] + }, + "author": "Cumulus Authors", + "license": "Apache-2.0", + "dependencies": { + "@cumulus/common": "^1.2.0", + "@cumulus/cumulus-message-adapter-js": "^1.0.1", + "@cumulus/ingest": "^1.2.0", + "babel-core": "^6.25.0", + "babel-loader": "^6.2.4", + "babel-plugin-transform-async-to-generator": "^6.24.1", + "babel-polyfill": "^6.23.0", + "babel-preset-es2015": "^6.24.1", + "json-loader": "~0.5.7", + "lodash.get": "^4.4.2", + "webpack": "~1.12.13" + }, + "devDependencies": { + "ava": "^0.23.0" + } +} diff --git a/cumulus/tasks/sf-sns-report/schemas/config.json b/cumulus/tasks/sf-sns-report/schemas/config.json new file mode 100644 index 00000000000..588d3df1dae --- /dev/null +++ b/cumulus/tasks/sf-sns-report/schemas/config.json @@ -0,0 +1,36 @@ +{ + "title": "SfSnsReportConfig", + "description": "Describes the config used by the sf-sns-report task", + "type": "object", + "additionalProperties": false, + "properties": { + "sfnEnd": { + "description": "indicate if it's the last step of the step function.", + "type": "boolean" + }, + "stack": { + "description": "the name of the deployment stack (from meta.stack). Required when sfnEnd is true and granule status is reported.", + "type": "string" + }, + "bucket": { + "description": "S3 bucket (from meta.buckets.internal). Required when sfnEnd is true and granule status is reported.", + "type": "string" + }, + "stateMachine": { + "description": "current state machine (from cumulus_meta.state_machine). Required when sfnEnd is true and granule status is reported.", + "type": "string" + }, + "executionName": { + "description": "execution name (from cumulus_meta.execution_name). Required when sfnEnd is true and granule status is reported.", + "type": "string" + } + }, + "oneOf": [ + { + "required": ["stack", "bucket", "stateMachine", "executionName"] + }, + { + "required": [] + } + ] +} diff --git a/cumulus/tasks/sf-sns-report/tests/.eslintrc.json b/cumulus/tasks/sf-sns-report/tests/.eslintrc.json new file mode 100644 index 00000000000..ada42bca77f --- /dev/null +++ b/cumulus/tasks/sf-sns-report/tests/.eslintrc.json @@ -0,0 +1,5 @@ +{ + "rules": { + "no-param-reassign": "off" + } +} diff --git a/cumulus/tasks/sf-sns-report/tests/index.js b/cumulus/tasks/sf-sns-report/tests/index.js new file mode 100644 index 00000000000..d34c7282b0a --- /dev/null +++ b/cumulus/tasks/sf-sns-report/tests/index.js @@ -0,0 +1,102 @@ +'use strict'; + +const test = require('ava'); +const { recursivelyDeleteS3Bucket, s3 } = require('@cumulus/common/aws'); +const { publishSnsMessage } = require('../index'); +const { cloneDeep, get } = require('lodash'); +const { randomString } = require('@cumulus/common/test-utils'); + +test('send report when sfn is running', (t) => { + const event = { + input: { + meta: { topic_arn: 'test_topic_arn' }, + anykey: 'anyvalue' + } + }; + + return publishSnsMessage(cloneDeep(event)) + .then((output) => { + t.not(get(output, 'MessageId', null)); + }); +}); + +test('send report when sfn is running with exception', (t) => { + const event = { + input: { + meta: { topic_arn: 'test_topic_arn' }, + exception: { + Error: 'TheError', + Cause: 'bucket not found' + }, + anykey: 'anyvalue' + } + }; + + return publishSnsMessage(cloneDeep(event)) + .catch((e) => { + t.is(e.message, event.input.exception.Cause); + }); +}); + +test('send report when sfn is running with TypeError', (t) => { + const event = { + input: { + meta: { topic_arn: 'test_topic_arn' }, + error: { + Error: 'TypeError', + Cause: 'resource not found' + }, + anykey: 'anyvalue' + } + }; + + return publishSnsMessage(cloneDeep(event)) + .catch((e) => { + t.is(e.message, event.input.error.Cause); + }); +}); + +test('send report when sfn is running with known error type', (t) => { + const event = { + input: { + meta: { topic_arn: 'test_topic_arn' }, + error: { + Error: 'PDRParsingError', + Cause: 'format error' + }, + anykey: 'anyvalue' + } + }; + + return publishSnsMessage(cloneDeep(event)) + .catch((e) => { + t.is(e.message, event.input.error.Cause); + }); +}); + +test('send report when sfn is finished and granule has succeeded', async (t) => { + const input = { + meta: { + topic_arn: 'test_topic_arn', + granuleId: randomString() + }, + anykey: 'anyvalue' + }; + const event = {}; + event.input = input; + event.config = {}; + event.config.sfnEnd = true; + event.config.stack = 'test_stack'; + event.config.bucket = randomString(); + event.config.stateMachine = + 'arn:aws:states:us-east-1:596205514787:stateMachine:TestCumulusParsePdrStateMach-K5Qk90fc8w4U'; + event.config.executionName = '7c543392-1da9-47f0-9c34-f43f6519412a'; + + await s3().createBucket({ Bucket: event.config.bucket }).promise(); + return publishSnsMessage(cloneDeep(event)) + .then((output) => { + t.not(get(output, 'MessageId', null)); + }) + .then(() => recursivelyDeleteS3Bucket(event.config.bucket)) + .catch(() => recursivelyDeleteS3Bucket(event.config.bucket).then(t.fail)); +}); diff --git a/cumulus/tasks/sf-sns-report/webpack.config.js b/cumulus/tasks/sf-sns-report/webpack.config.js new file mode 100644 index 00000000000..f0d512ee35d --- /dev/null +++ b/cumulus/tasks/sf-sns-report/webpack.config.js @@ -0,0 +1,19 @@ +module.exports = { + entry: ['babel-polyfill', './index.js'], + output: { + libraryTarget: 'commonjs2', + filename: 'dist/index.js' + }, + target: 'node', + devtool: 'sourcemap', + module: { + loaders: [{ + test: /\.js?$/, + exclude: /node_modules(?!\/@cumulus)/, + loader: 'babel' + }, { + test: /\.json$/, + loader: 'json' + }] + } +}; diff --git a/cumulus/tasks/sync-granule/package.json b/cumulus/tasks/sync-granule/package.json index d4f6fc8fb1a..aed87eb41c6 100644 --- a/cumulus/tasks/sync-granule/package.json +++ b/cumulus/tasks/sync-granule/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sync-granule", - "version": "1.1.4", + "version": "1.2.0", "description": "Download a given granule", "main": "index.js", "directories": { @@ -14,7 +14,7 @@ "access": "public" }, "scripts": { - "test": "env TEST=true ava", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" @@ -41,10 +41,10 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.4", - "@cumulus/test-data": "^1.1.3", + "@cumulus/ingest": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", @@ -55,7 +55,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "fs-extra": "^5.0.0", "lodash": "^4.17.5", "proxyquire": "^1.8.0", diff --git a/cumulus/tasks/sync-http-urls/package.json b/cumulus/tasks/sync-http-urls/package.json index 3dcd81bba3d..950f3ed42b1 100644 --- a/cumulus/tasks/sync-http-urls/package.json +++ b/cumulus/tasks/sync-http-urls/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/sync-http-urls", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Synchronizes http links to s3", "main": "index.js", "keywords": [ @@ -14,7 +14,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "async": "^2.0.0", "lodash": "^4.13.1", "request": "^2.83.0" diff --git a/cumulus/tasks/sync-wms/package.json b/cumulus/tasks/sync-wms/package.json index c1bdb47f02c..01290500409 100644 --- a/cumulus/tasks/sync-wms/package.json +++ b/cumulus/tasks/sync-wms/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/sync-wms", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Transforms input granule info to WMS URLs for URL sync", "main": "index.js", "keywords": [ @@ -14,6 +14,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3" + "@cumulus/common": "^1.2.0" } } diff --git a/cumulus/tasks/tee/package.json b/cumulus/tasks/tee/package.json index f81306182ed..a394753ad74 100644 --- a/cumulus/tasks/tee/package.json +++ b/cumulus/tasks/tee/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/tee", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Task for testing that splits output", "main": "index.js", "keywords": [ @@ -15,7 +15,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", diff --git a/cumulus/tasks/trigger-ingest/package.json b/cumulus/tasks/trigger-ingest/package.json index 9f1890d599c..2863629e6ea 100644 --- a/cumulus/tasks/trigger-ingest/package.json +++ b/cumulus/tasks/trigger-ingest/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/trigger-ingest", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Transforms input granule info to WMS URLs for URL sync", "main": "index.js", "keywords": [ @@ -14,7 +14,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "lodash": "^4.17.4" } } diff --git a/cumulus/tasks/trigger-mrf-gen/package.json b/cumulus/tasks/trigger-mrf-gen/package.json index e9a6b832683..801e0ac3e3b 100644 --- a/cumulus/tasks/trigger-mrf-gen/package.json +++ b/cumulus/tasks/trigger-mrf-gen/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/trigger-mrf-gen", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Invokes MRFGen for every group of images passed to it", "main": "index.js", "keywords": [ @@ -32,12 +32,12 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3" + "@cumulus/common": "^1.2.0" }, "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/trigger-process-pdrs/package.json b/cumulus/tasks/trigger-process-pdrs/package.json index 885ef579140..745ad55801e 100644 --- a/cumulus/tasks/trigger-process-pdrs/package.json +++ b/cumulus/tasks/trigger-process-pdrs/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/trigger-process-pdrs", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "Transforms input granule info to WMS URLs for URL sync", "main": "index.js", "keywords": [ @@ -34,12 +34,12 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3" + "@cumulus/common": "^1.2.0" }, "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/validate-archives/package.json b/cumulus/tasks/validate-archives/package.json index 2c6907c404c..8ba6add2368 100644 --- a/cumulus/tasks/validate-archives/package.json +++ b/cumulus/tasks/validate-archives/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/validate-archives", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "SIPS handler archive validation task", "main": "index.js", "keywords": [ @@ -35,7 +35,7 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "checksum": "^0.1.1", "ftp": "^0.3.10", "gunzip-maybe": "^1.4.1", @@ -50,7 +50,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/cumulus/tasks/validate-pdr/package.json b/cumulus/tasks/validate-pdr/package.json index 194a6f98bca..8429e8e0b72 100644 --- a/cumulus/tasks/validate-pdr/package.json +++ b/cumulus/tasks/validate-pdr/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/validate-pdr", "private": true, - "version": "1.1.4", + "version": "1.2.0", "description": "SIPS handler PDR processing task", "main": "index.js", "keywords": [ @@ -18,12 +18,10 @@ "test": "test" }, "scripts": { - "test": "env TEST=true ava" + "test": "ava" }, "ava": { - "files": [ - "test" - ], + "files": "test", "babel": "inherit", "require": [ "babel-polyfill", @@ -39,9 +37,9 @@ ] }, "dependencies": { - "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.4", - "@cumulus/pvl": "^1.0.0", + "@cumulus/common": "^1.2.0", + "@cumulus/ingest": "^1.2.0", + "@cumulus/pvl": "^1.2.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", @@ -53,7 +51,7 @@ "devDependencies": { "@ava/babel-preset-stage-4": "^1.1.0", "@ava/babel-preset-transform-test-files": "^3.0.0", - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-polyfill": "^6.23.0", diff --git a/lerna.json b/lerna.json index 4022c39ff8c..ea04ef403b4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "2.9.0", - "version": "1.1.4", + "version": "1.2.0", "packages": [ "packages/*", "cumulus/tasks/*", diff --git a/package.json b/package.json index 43ea36231c9..bb7ee36041b 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "Cumulus Framework for ingesting and processing Nasa Earth data streams", "scripts": { - "e2e": "env TEST=true ava tests/*.js --serial", + "e2e": "ava tests/*.js --serial", "test": "lerna run test", "bootstrap": "lerna bootstrap", "ybootstrap": "lerna bootstrap --npm-client=yarn", @@ -44,14 +44,14 @@ "devDependencies": { "ava": "^0.25.0", "babel-core": "^6.13.2", - "babel-eslint": "^6.1.2", + "babel-eslint": "^8.2.2", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.8.0", "babel-polyfill": "^6.13.0", "babel-preset-es2015": "^6.13.2", "babel-preset-es2017": "^6.24.1", "copy-webpack-plugin": "^4.0.1", - "eslint": "^3.2.2", + "eslint": "^4.18.2", "eslint-config-airbnb": "^10.0.0", "eslint-plugin-import": "^1.13.0", "eslint-plugin-jsdoc": "^3.3.1", diff --git a/packages/api/config/lambdas.yml b/packages/api/config/lambdas.yml index 24b8afaddfa..c5b8699e34c 100644 --- a/packages/api/config/lambdas.yml +++ b/packages/api/config/lambdas.yml @@ -61,9 +61,12 @@ kinesisConsumer: RulesTable: function: Ref value: RulesTableDynamoDB - invoke: + CollectionsTable: function: Ref - value: ScheduleSFLambdaFunction + value: CollectionsTableDynamoDB + ProvidersTable: + function: Ref + value: ProvidersTableDynamoDB bucket: '{{buckets.internal}}' ScheduleSF: diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index ddd8f3b7bcc..c9161be8ae5 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -2,6 +2,7 @@ const _get = require('lodash.get'); const { justLocalRun } = require('@cumulus/common/local-helpers'); +const { inTestMode } = require('@cumulus/common/test-utils'); const log = require('@cumulus/common/log'); const { handle } = require('../lib/response'); const models = require('../models'); @@ -122,7 +123,7 @@ function handler(event, context) { return context.fail('HttpMethod is missing'); } - return handle(event, context, !process.env.TEST /* authCheck */, cb => { + return handle(event, context, !inTestMode() /* authCheck */, cb => { if (event.httpMethod === 'GET' && event.pathParameters) { get(event, cb); } diff --git a/packages/api/endpoints/providers.js b/packages/api/endpoints/providers.js index 0fa506e4e8d..46e8bf3b4b9 100644 --- a/packages/api/endpoints/providers.js +++ b/packages/api/endpoints/providers.js @@ -2,6 +2,7 @@ 'use strict'; const _get = require('lodash.get'); +const { inTestMode } = require('@cumulus/common/test-utils'); const { handle } = require('../lib/response'); const models = require('../models'); const RecordDoesNotExist = require('../lib/errors').RecordDoesNotExist; @@ -104,7 +105,7 @@ function del(event, cb) { } function handler(event, context) { - handle(event, context, !process.env.TEST /* authCheck */, (cb) => { + handle(event, context, !inTestMode() /* authCheck */, (cb) => { if (event.httpMethod === 'GET' && event.pathParameters) { get(event, cb); } diff --git a/packages/api/endpoints/rules.js b/packages/api/endpoints/rules.js index 5c78f005d02..394f93d4d23 100644 --- a/packages/api/endpoints/rules.js +++ b/packages/api/endpoints/rules.js @@ -3,6 +3,7 @@ const _get = require('lodash.get'); const { justLocalRun } = require('@cumulus/common/local-helpers'); +const { inTestMode } = require('@cumulus/common/test-utils'); const { handle } = require('../lib/response'); const models = require('../models'); const { RecordDoesNotExist } = require('../lib/errors'); @@ -118,7 +119,7 @@ async function del(event) { } function handler(event, context) { - return handle(event, context, !process.env.TEST /* authCheck */, cb => { + return handle(event, context, !inTestMode() /* authCheck */, cb => { if (event.httpMethod === 'GET' && event.pathParameters) { get(event, cb); } diff --git a/packages/api/es/search.js b/packages/api/es/search.js index b2e00297c19..7f576c414e9 100644 --- a/packages/api/es/search.js +++ b/packages/api/es/search.js @@ -10,6 +10,7 @@ const omit = require('lodash.omit'); const aws = require('aws-sdk'); const httpAwsEs = require('http-aws-es'); const elasticsearch = require('elasticsearch'); +const { inTestMode } = require('@cumulus/common/test-utils'); const queries = require('./queries'); const aggs = require('./aggregations'); @@ -23,7 +24,7 @@ class BaseSearch { let esConfig; // this is needed for getting temporary credentials from IAM role - if (process.env.TEST) { + if (inTestMode()) { if (!process.env.LOCALSTACK_HOST) { throw new Error('The LOCALSTACK_HOST environment variable is not set.'); } diff --git a/packages/api/lambdas/bootstrap.js b/packages/api/lambdas/bootstrap.js index d505e335b2a..5ca47596337 100644 --- a/packages/api/lambdas/bootstrap.js +++ b/packages/api/lambdas/bootstrap.js @@ -10,6 +10,7 @@ * - creating API users * - encrypting CMR user/pass and adding it to configuration files */ + 'use strict'; const https = require('https'); @@ -44,19 +45,17 @@ async function bootstrapElasticSearch(host, index = 'cumulus') { else { log.info(`index ${index} already exists`); } - - return; } async function bootstrapUsers(table, records) { if (!table) { - return new Promise(resolve => resolve()); + return new Promise((resolve) => resolve()); } const user = new Manager(table); // delete all user records const existingUsers = await user.scan(); - await Promise.all(existingUsers.Items.map(u => user.delete({ userName: u.userName }))); + await Promise.all(existingUsers.Items.map((u) => user.delete({ userName: u.userName }))); // add new ones const additions = records.map((record) => user.create({ userName: record.username, @@ -69,7 +68,7 @@ async function bootstrapUsers(table, records) { async function bootstrapCmrProvider(password) { if (!password) { - return new Promise(resolve => resolve('nopassword')); + return new Promise((resolve) => resolve('nopassword')); } return DefaultProvider.encrypt(password); } @@ -140,7 +139,7 @@ function handler(event, context, cb) { }; return sendResponse(event, 'SUCCESS', data, cb); - }).catch(e => { + }).catch((e) => { log.error(e); return sendResponse(event, 'FAILED', null, cb); }); @@ -155,8 +154,8 @@ justLocalRun(() => { //const a = {}; //handler(a, {}, (e, r) => console.log(e, r)); //bootstrapCmrProvider('testing').then(r => { - //console.log(r) - //return DefaultProvider.decrypt(r) + //console.log(r) + //return DefaultProvider.decrypt(r) //}).then(r => console.log(r)) - //.catch(e => console.log(e)); + //.catch(e => console.log(e)); }); diff --git a/packages/api/lambdas/db-indexer.js b/packages/api/lambdas/db-indexer.js index 1ea58dddff3..a2a8a936ab0 100644 --- a/packages/api/lambdas/db-indexer.js +++ b/packages/api/lambdas/db-indexer.js @@ -17,7 +17,7 @@ function indexRecord(esClient, record) { //determine whether the record should be indexed const acceptedTables = ['Collection', 'Provider', 'Rule']; - const tableConfig = {} + const tableConfig = {}; acceptedTables.forEach((a) => { tableConfig[`${stack}-${a}sTable`] = indexer[`index${a}`]; }); @@ -54,21 +54,20 @@ function indexRecord(esClient, record) { } async function indexRecords(records) { - const concurrencyLimit = process.env.CONCURRENCY || 3 + const concurrencyLimit = process.env.CONCURRENCY || 3; const limit = pLimit(concurrencyLimit); const esClient = await Search.es(); - const promises = records.map((record) => limit( - () => indexRecord(esClient, record) - )); + const promises = records.map((record) => limit(() => indexRecord(esClient, record))); return Promise.all(promises); } /** * Sync changes to dynamodb to an elasticsearch instance. * Sending updates to this lambda is handled by automatically AWS. - * @param {array} Records list of records with an eventName property signifying REMOVE or INSERT. - * @return {string} response text indicating the number of records altered in elasticsearch. + * + * @param {Array} Records - list of records with an eventName property signifying REMOVE or INSERT. + * @returns {string} response text indicating the number of records altered in elasticsearch. */ function handler(event, context, cb) { const records = event.Records; diff --git a/packages/api/lambdas/jobs.js b/packages/api/lambdas/jobs.js index 1cd80f4d82c..7c38385772a 100644 --- a/packages/api/lambdas/jobs.js +++ b/packages/api/lambdas/jobs.js @@ -1,4 +1,5 @@ /* runs a bunch of periodic jobs to keep the database updateToDate */ + 'use strict'; const get = require('lodash.get'); @@ -19,8 +20,8 @@ async function findStaleRecords(type, q, limit = 100, page = 1) { const response = await search.query(); //if (response.results.length >= limit) { - //const more = await findStaleRecords(type, q, limit, page + 1); - //return response.results.concat(more); + //const more = await findStaleRecords(type, q, limit, page + 1); + //return response.results.concat(more); //} return response.results; } @@ -29,16 +30,12 @@ async function updateGranulesAndPdrs(esClient, url, error) { // find related granule and update their status let searchTerm = `execution:"${url}"`; const granules = await findStaleRecords('granule', searchTerm, 100); - await Promise.all(granules.map(g => partialRecordUpdate( - esClient, g.granuleId, 'granule', { status: 'failed', error }, g.collectionId - ))); + await Promise.all(granules.map((g) => partialRecordUpdate(esClient, g.granuleId, 'granule', { status: 'failed', error }, g.collectionId))); // find related pdrs and update their status searchTerm = `execution:"${url}"`; const pdrs = await findStaleRecords('pdr', searchTerm, 100); - await Promise.all(pdrs.map(p => partialRecordUpdate( - esClient, p.pdrName, 'pdr', { status: 'failed', error } - ))); + await Promise.all(pdrs.map((p) => partialRecordUpdate(esClient, p.pdrName, 'pdr', { status: 'failed', error }))); } async function checkExecution(arn, url, timestamp, esClient) { @@ -133,17 +130,11 @@ async function cleanup() { const limit = pLimit(2); - await Promise.all( - executions.slice(0, 400).map( - ex => limit( - () => checkExecution(ex.arn, ex.execution, ex.timestamp, esClient) - ) - ) - ); + await Promise.all(executions.slice(0, 400).map((ex) => limit(() => checkExecution(ex.arn, ex.execution, ex.timestamp, esClient)))); } function handler(event, context, cb) { - cleanup().then(() => cb()).catch(e => { + cleanup().then(() => cb()).catch((e) => { log.error(e); cb(e); }); diff --git a/packages/api/lambdas/kinesis-consumer.js b/packages/api/lambdas/kinesis-consumer.js index c00da611b4f..acb7a4ba238 100644 --- a/packages/api/lambdas/kinesis-consumer.js +++ b/packages/api/lambdas/kinesis-consumer.js @@ -1,10 +1,13 @@ /* eslint-disable require-yield */ + 'use strict'; + const Ajv = require('ajv'); +const log = require('@cumulus/common/log'); const Rule = require('../models/rules'); const messageSchema = require('./kinesis-consumer-event-schema.json'); -const { queueWorkflowMessage } = require('@cumulus/ingest/queue'); +const sfSchedule = require('./sf-scheduler'); /** * `getKinesisRules` scans and returns DynamoDB rules table for enabled, @@ -14,7 +17,7 @@ const { queueWorkflowMessage } = require('@cumulus/ingest/queue'); * @returns {Array} List of zero or more rules found from table scan */ async function getKinesisRules(event) { - const collection = event.collection; + const { collection } = event; const model = new Rule(); const kinesisRules = await model.scan({ names: { @@ -51,8 +54,11 @@ async function queueMessageForRule(kinesisRule, eventObject) { payload: eventObject }; - return Rule.buildPayload(item) - .then(queueWorkflowMessage); + const payload = await Rule.buildPayload(item); + return new Promise((resolve, reject) => sfSchedule(payload, {}, (err, result) => { + if (err) reject(err); + resolve(result); + })); } /** @@ -63,10 +69,10 @@ async function queueMessageForRule(kinesisRule, eventObject) { * @returns {(error|Object)} Throws an Ajv.ValidationError if event object is invalid. * Returns the event object if event is valid. */ -async function validateMessage(event) { +function validateMessage(event) { const ajv = new Ajv({ allErrors: true }); const validate = ajv.compile(messageSchema); - return await validate(event); + return validate(event); } /** @@ -86,7 +92,12 @@ function processRecord(record) { .then(getKinesisRules) .then((kinesisRules) => ( Promise.all(kinesisRules.map((kinesisRule) => queueMessageForRule(kinesisRule, eventObject))) - )); + )) + .catch((err) => { + log.error('Caught error in process record:'); + log.error(err); + return err; + }); } /** diff --git a/packages/api/lambdas/sf-scheduler.js b/packages/api/lambdas/sf-scheduler.js index 9b6b9257e41..a42615cf242 100644 --- a/packages/api/lambdas/sf-scheduler.js +++ b/packages/api/lambdas/sf-scheduler.js @@ -10,9 +10,10 @@ const { Provider, Collection } = require('../models'); * Builds a cumulus-compatible message and adds it to the startSF queue * startSF queue will then start a stepfunction for the given message * - * @param {object} event lambda input message - * @param {object} context lambda context - * @param {function} cb lambda callback + * @param {Object} event - lambda input message + * @param {Object} context - lambda context + * @param {function} cb - lambda callback + * @returns {function} Calls callback with result of SQS.sendMessage or error */ function schedule(event, context, cb) { const template = get(event, 'template'); @@ -53,9 +54,11 @@ function schedule(event, context, cb) { .then((c) => { if (c) message.meta.collection = c; }) - .then(() => SQS.sendMessage(message.meta.queues.startSF, message)) + .then(() => { + SQS.sendMessage(message.meta.queues.startSF, message); + }) .then((r) => cb(null, r)) - .catch(e => cb(e)); + .catch((e) => cb(e)); } module.exports = schedule; diff --git a/packages/api/package.json b/packages/api/package.json index 0922d2384c9..409cb888c5f 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,21 +1,22 @@ { "name": "@cumulus/api", - "version": "1.1.4", + "version": "1.2.0", "description": "Lambda functions for handling all daac's API operations", "main": "index.js", "scripts": { - "test": "TEST=true ava tests/test-*.js --serial", + "test": "ava", "build": "webpack --progress", "watch": "webpack --progress -w", "postinstall": "npm run build" }, "ava": { - "files": "tests", + "files": "tests/test-*.js", "babel": "inherit", "require": [ "babel-polyfill", "babel-register" - ] + ], + "serial": true }, "publishConfig": { "access": "public" @@ -32,10 +33,10 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/cmrjs": "^1.1.3", - "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.4", - "@cumulus/pvl": "^1.0.0", + "@cumulus/cmrjs": "^1.2.0", + "@cumulus/common": "^1.2.0", + "@cumulus/ingest": "^1.2.0", + "@cumulus/pvl": "^1.2.0", "ajv": "^5.2.2", "archiver": "^2.1.1", "aws-sdk": "^2.95.0", @@ -57,7 +58,7 @@ "lodash.merge": "^4.5.0", "lodash.omit": "^4.5.0", "lodash.uniqby": "^4.7.0", - "moment": "^2.18.1", + "moment": "^2.21.0", "node-forge": "^0.7.1", "p-limit": "^1.1.0", "querystring": "^0.2.0", @@ -65,7 +66,7 @@ "webpack": "^1.15.0" }, "devDependencies": { - "ava": "^0.21.0", + "ava": "^0.23.0", "sinon": "^2.0.0-pre.5" } } diff --git a/packages/api/tests/test-db-indexer.js b/packages/api/tests/test-db-indexer.js index 60306c98221..ce82353b773 100644 --- a/packages/api/tests/test-db-indexer.js +++ b/packages/api/tests/test-db-indexer.js @@ -74,7 +74,6 @@ if (process.env.LOCALSTACK_HOST === 'localhost') { }, Environment: { Variables: { - 'TEST': 'true', 'LOCALSTACK_HOST': process.env.DOCKERHOST, 'stackName': process.env.stackName } diff --git a/packages/api/tests/test-kinesis-consumer.js b/packages/api/tests/test-kinesis-consumer.js index 9418bd50e97..feac904b977 100644 --- a/packages/api/tests/test-kinesis-consumer.js +++ b/packages/api/tests/test-kinesis-consumer.js @@ -1,14 +1,18 @@ 'use strict'; -const test = require('ava'); -const sinon = require('sinon'); const get = require('lodash.get'); -const { sqs, s3, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); -const { createQueue, randomString } = require('@cumulus/common/test-utils'); +const sinon = require('sinon'); +const test = require('ava'); +const { randomString } = require('@cumulus/common/test-utils'); +const { SQS } = require('@cumulus/ingest/aws'); +const { s3, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); const { getKinesisRules, handler } = require('../lambdas/kinesis-consumer'); + const manager = require('../models/base'); +const Collection = require('../models/collections'); const Rule = require('../models/rules'); +const Provider = require('../models/providers'); const testCollectionName = 'test-collection'; const ruleTableParams = { @@ -23,17 +27,20 @@ const eventData = JSON.stringify({ const event = { Records: [ - { kinesis: { data: new Buffer(eventData).toString('base64') } }, - { kinesis: { data: new Buffer(eventData).toString('base64') } } + { kinesis: { data: Buffer.from(eventData).toString('base64') } }, + { kinesis: { data: Buffer.from(eventData).toString('base64') } } ] }; +const collection = { + name: testCollectionName, + version: '0.0.0' +}; +const provider = { id: 'PROV1' }; + const commonRuleParams = { - collection: { - name: testCollectionName, - version: '0.0.0' - }, - provider: 'PROV1', + collection, + provider: provider.id, rule: { type: 'kinesis', value: 'test-kinesisarn' @@ -69,36 +76,39 @@ function testCallback(err, object) { return object; } +let sfSchedulerSpy; +const stubQueueUrl = 'stubQueueUrl'; + test.beforeEach(async (t) => { + sfSchedulerSpy = sinon.stub(SQS, 'sendMessage').returns(true); t.context.templateBucket = randomString(); - await s3().createBucket({ Bucket: t.context.templateBucket }).promise(); - t.context.stateMachineArn = randomString(); + const messageTemplateKey = `${randomString()}/template.json`; - t.context.queueUrl = await createQueue(); - + t.context.messageTemplateKey = messageTemplateKey; t.context.messageTemplate = { cumulus_meta: { state_machine: t.context.stateMachineArn }, - meta: { queues: { startSF: t.context.queueUrl } } + meta: { queues: { startSF: stubQueueUrl } } }; - const messageTemplateKey = `${randomString()}/template.json`; + + await s3().createBucket({ Bucket: t.context.templateBucket }).promise(); await s3().putObject({ Bucket: t.context.templateBucket, Key: messageTemplateKey, Body: JSON.stringify(t.context.messageTemplate) }).promise(); - sinon.stub(Rule, 'buildPayload').callsFake((item) => - Promise.resolve({ - template: `s3://${t.context.templateBucket}/${messageTemplateKey}`, - provider: item.provider, - collection: item.collection, - meta: get(item, 'meta', {}), - payload: get(item, 'payload', {}) - }) - ); + sinon.stub(Rule, 'buildPayload').callsFake((item) => Promise.resolve({ + template: `s3://${t.context.templateBucket}/${messageTemplateKey}`, + provider: item.provider, + collection: item.collection, + meta: get(item, 'meta', {}), + payload: get(item, 'payload', {}) + })); + sinon.stub(Provider.prototype, 'get').returns(provider); + sinon.stub(Collection.prototype, 'get').returns(collection); t.context.tableName = randomString(); process.env.RulesTable = t.context.tableName; @@ -115,16 +125,18 @@ test.beforeEach(async (t) => { test.afterEach(async (t) => { await Promise.all([ recursivelyDeleteS3Bucket(t.context.templateBucket), - sqs().deleteQueue({ QueueUrl: t.context.queueUrl }).promise(), manager.deleteTable(t.context.tableName) ]); + sfSchedulerSpy.restore(); Rule.buildPayload.restore(); + Provider.prototype.get.restore(); + Collection.prototype.get.restore(); }); // getKinesisRule tests // eslint-disable-next-line max-len -test('it should look up kinesis-type rules which are associated with the collection, but not those that are disabled', (t) => { - return getKinesisRules(JSON.parse(eventData)) +test('it should look up kinesis-type rules which are associated with the collection, but not those that are disabled', async (t) => { + await getKinesisRules(JSON.parse(eventData)) .then((result) => { t.is(result.length, 2); }); @@ -133,49 +145,53 @@ test('it should look up kinesis-type rules which are associated with the collect // handler tests test('it should enqueue a message for each associated workflow', async (t) => { await handler(event, {}, testCallback); - await sqs().receiveMessage({ - QueueUrl: t.context.queueUrl, - MaxNumberOfMessages: 10, - WaitTimeSeconds: 1 - }).promise() - .then((receiveMessageResponse) => { - t.is(receiveMessageResponse.Messages.length, 4); - receiveMessageResponse.Messages.map((message) => ( - t.is(JSON.stringify(JSON.parse(message.Body).payload), JSON.stringify({ collection: 'test-collection' })) - )); - }); + const actualQueueUrl = sfSchedulerSpy.getCall(0).args[0]; + t.is(actualQueueUrl, stubQueueUrl); + const actualMessage = sfSchedulerSpy.getCall(0).args[1]; + const expectedMessage = { + cumulus_meta: { + state_machine: t.context.stateMachineArn + }, + meta: { + queues: { startSF: stubQueueUrl }, + provider, + collection + }, + payload: { + collection: 'test-collection' + } + }; + t.is(actualMessage.cumulus_meta.state_machine, expectedMessage.cumulus_meta.state_machine); + t.deepEqual(actualMessage.meta, expectedMessage.meta); + t.deepEqual(actualMessage.payload, expectedMessage.payload); }); -test('it should throw an error if message does not include a collection', (t) => { +test('it should throw an error if message does not include a collection', async (t) => { const invalidMessage = JSON.stringify({}); const kinesisEvent = { - Records: [{ kinesis: { data: new Buffer(invalidMessage).toString('base64') } }] + Records: [{ kinesis: { data: Buffer.from(invalidMessage).toString('base64') } }] }; - return handler(kinesisEvent, {}, testCallback) - .catch((err) => { - const errObject = JSON.parse(err); - t.is(errObject.errors[0].dataPath, ''); - t.is(errObject.errors[0].message, 'should have required property \'collection\''); - }); + const errors = await handler(kinesisEvent, {}, testCallback); + t.is(errors[0].message, 'validation failed'); + t.is(errors[0].errors[0].dataPath, ''); + t.is(errors[0].errors[0].message, 'should have required property \'collection\''); }); -test('it should throw an error if message collection has wrong data type', (t) => { +test('it should throw an error if message collection has wrong data type', async (t) => { const invalidMessage = JSON.stringify({ collection: {} }); const kinesisEvent = { - Records: [{ kinesis: { data: new Buffer(invalidMessage).toString('base64') } }] + Records: [{ kinesis: { data: Buffer.from(invalidMessage).toString('base64') } }] }; - return handler(kinesisEvent, {}, testCallback) - .catch((err) => { - const errObject = JSON.parse(err); - t.is(errObject.errors[0].dataPath, '.collection'); - t.is(errObject.errors[0].message, 'should be string'); - }); + const errors = await handler(kinesisEvent, {}, testCallback); + t.is(errors[0].message, 'validation failed'); + t.is(errors[0].errors[0].dataPath, '.collection'); + t.is(errors[0].errors[0].message, 'should be string'); }); test('it should not throw if message is valid', (t) => { const validMessage = JSON.stringify({ collection: 'confection-collection' }); const kinesisEvent = { - Records: [{ kinesis: { data: new Buffer(validMessage).toString('base64') } }] + Records: [{ kinesis: { data: Buffer.from(validMessage).toString('base64') } }] }; return handler(kinesisEvent, {}, testCallback).then((r) => t.deepEqual(r, [[]])); }); diff --git a/packages/cmrjs/package.json b/packages/cmrjs/package.json index 56c0b6ae91c..435cd524b10 100644 --- a/packages/cmrjs/package.json +++ b/packages/cmrjs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/cmrjs", - "version": "1.1.3", + "version": "1.2.0", "description": "A node SDK for CMR", "scripts": { "test": "echo 'no tests'" @@ -26,7 +26,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "got": "^7.1.0", "json-loader": "^0.5.4", "lodash.property": "^4.4.2", diff --git a/packages/common/aws.js b/packages/common/aws.js index c6fe7460e7c..702bdb84343 100644 --- a/packages/common/aws.js +++ b/packages/common/aws.js @@ -8,7 +8,7 @@ const path = require('path'); const url = require('url'); const log = require('./log'); const string = require('./string'); -const testUtils = require('./test-utils'); +const { inTestMode, randomString, testAwsClient } = require('./test-utils'); const promiseRetry = require('promise-retry'); const region = exports.region = process.env.AWS_DEFAULT_REGION || 'us-east-1'; @@ -44,11 +44,11 @@ const awsClient = (Service, version = null) => { const options = {}; if (version) options.apiVersion = version; - if (process.env.TEST) { + if (inTestMode()) { if (AWS.DynamoDB.DocumentClient.serviceIdentifier === undefined) { AWS.DynamoDB.DocumentClient.serviceIdentifier = 'dynamodb'; } - return memoize(() => testUtils.testAwsClient(Service, options)); + return memoize(() => testAwsClient(Service, options)); } return memoize(() => new Service(options)); }; @@ -64,6 +64,7 @@ exports.dynamodbstreams = awsClient(AWS.DynamoDBStreams, '2012-08-10'); exports.dynamodbDocClient = awsClient(AWS.DynamoDB.DocumentClient, '2012-08-10'); exports.sfn = awsClient(AWS.StepFunctions, '2016-11-23'); exports.cf = awsClient(AWS.CloudFormation, '2010-05-15'); +exports.sns = awsClient(AWS.SNS, '2010-03-31'); /** * Describes the resources belonging to a given CloudFormation stack @@ -480,6 +481,34 @@ exports.fromSfnExecutionName = (str, delimiter = '__') => .map((s) => s.replace(/!/g, '\\').replace('"', '\\"')) .map((s) => JSON.parse(`"${s}"`)); +/** + * Create an SQS Queue. Properly handles localstack queue URLs + * + * @param {string} queueName - defaults to a random string + * @returns {Promise.} the Queue URL + */ +async function createQueue(queueName) { + const actualQueueName = queueName || randomString(); + + const createQueueResponse = await exports.sqs().createQueue({ + QueueName: actualQueueName + }).promise(); + + if (inTestMode()) { + // Properly set the Queue URL. This is needed because LocalStack always + // returns the QueueUrl as "localhost", even if that is not where it should + // actually be found. CircleCI breaks without this. + const returnedQueueUrl = url.parse(createQueueResponse.QueueUrl); + returnedQueueUrl.host = undefined; + returnedQueueUrl.hostname = process.env.LOCALSTACK_HOST; + + return url.format(returnedQueueUrl); + } + + return createQueueResponse.QueueUrl; +} +exports.createQueue = createQueue; + /** * Send a message to AWS SQS * @@ -575,14 +604,15 @@ exports.getGranuleS3Params = (granuleId, stack, bucket) => { /** * Set the status of a granule +* * @name setGranuleStatus -* @param {string} granuleId -* @param {string} stack = the deployment stackname +* @param {string} granuleId - granule id +* @param {string} stack - the deployment stackname * @param {string} bucket - the deployment bucket name -* @param {string} stateMachineArn -* @param {string} executionName -* @param {string} status -* @return {promise} returns the response from `S3.put` as a promise +* @param {string} stateMachineArn - statemachine arn +* @param {string} executionName - execution name +* @param {string} status - granule status +* @returns {Promise} returns the response from `S3.put` as a promise **/ exports.setGranuleStatus = async ( granuleId, @@ -594,5 +624,7 @@ exports.setGranuleStatus = async ( ) => { const key = exports.getGranuleS3Params(granuleId, stack, bucket); const executionArn = exports.getExecutionArn(stateMachineArn, executionName); - await exports.s3().putObject(bucket, key, '', null, { executionArn, status }).promise(); + const params = { Bucket: bucket, Key: key }; + params.Metadata = { executionArn, status }; + await exports.s3().putObject(params).promise(); }; diff --git a/packages/common/package.json b/packages/common/package.json index 0ec87400263..2484345ab73 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/common", - "version": "1.1.3", + "version": "1.2.0", "description": "Common utilities used across tasks", "keywords": [ "GIBS", @@ -18,7 +18,7 @@ "lint": "eslint .", "mocha": "mocha-webpack test/**/*.js", "mocha-junit": "npm run mocha -- --reporter mocha-junit-reporter --reporter-options mochaFile=test-results.xml", - "test": "npm run mocha && env TEST=true ava", + "test": "npm run mocha && ava", "test-ci": "npm run lint && npm run mocha-junit" }, "ava": { @@ -44,17 +44,17 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/test-data": "^1.1.3", + "@cumulus/test-data": "^1.2.0", "ajv": "^5.0.4-beta.3", "ajv-cli": "^1.1.1", "async": "^2.0.0", "aws-sdk": "^2.4.11", "babel-core": "^6.13.2", - "babel-eslint": "^6.1.2", + "babel-eslint": "^8.2.2", "babel-polyfill": "^6.13.0", "babel-preset-env": "^1.6.1", "cwait": "^1.0.0", - "eslint": "^3.2.2", + "eslint": "^4.18.2", "eslint-config-airbnb": "^10.0.0", "eslint-plugin-import": "^1.13.0", "eslint-plugin-jsx-a11y": "^2.1.0", @@ -66,7 +66,7 @@ "glob": "^7.1.1", "js-yaml": "^3.8.3", "lodash": "^4.13.1", - "mocha": "^2.5.3", + "mocha": "^5.0.4", "mocha-junit-reporter": "^1.11.1", "mocha-webpack": "^0.7.0", "prepend-loader": "0.0.2", diff --git a/packages/common/test-utils.js b/packages/common/test-utils.js index 0aed764495a..cab19fee7bf 100644 --- a/packages/common/test-utils.js +++ b/packages/common/test-utils.js @@ -4,10 +4,10 @@ const Ajv = require('ajv'); const crypto = require('crypto'); const path = require('path'); -const url = require('url'); -const aws = require('./aws'); const fs = require('fs-extra'); +exports.inTestMode = () => process.env.NODE_ENV === 'test'; + /** * Generate a 40-character random string * @@ -40,7 +40,8 @@ const localStackPorts = { * Test if a given AWS service is supported by LocalStack. * * @param {Function} Service - an AWS service object constructor function - * @returns {boolean} + * @returns {boolean} true or false depending on whether the service is + * supported by LocalStack */ function localstackSupportedService(Service) { const serviceIdentifier = Service.serviceIdentifier; @@ -78,8 +79,6 @@ function localStackAwsClient(Service, options) { /** * Create an AWS service object that does not actually talk to AWS. * - * @todo Update this to return a mock AWS client if not supported by localstack - * * @param {Function} Service - an AWS service object constructor function * @param {Object} options - options to pass to the service object constructor function * @returns {Object} - an AWS service object @@ -92,27 +91,6 @@ function testAwsClient(Service, options) { } exports.testAwsClient = testAwsClient; -/** - * Create an SQS queue for testing - * - * @returns {string} - an SQS queue URL - */ -async function createQueue() { - const createQueueResponse = await aws.sqs().createQueue({ - QueueName: exports.randomString() - }).promise(); - - // Properly set the Queue URL. This is needed because LocalStack always - // returns the QueueUrl as "localhost", even if that is not where it should - // actually be found. CircleCI breaks without this. - const returnedQueueUrl = url.parse(createQueueResponse.QueueUrl); - returnedQueueUrl.host = undefined; - returnedQueueUrl.hostname = process.env.LOCALSTACK_HOST; - - return url.format(returnedQueueUrl); -} -exports.createQueue = createQueue; - /** * Validate an object using json-schema * @@ -183,6 +161,8 @@ exports.validateOutput = validateOutput; /** * Determine the path of the current git repo * + * @param {string} dirname - the directory that you're trying to find the git + * root for * @returns {Promise.} - the filesystem path of the current git repo */ async function findGitRepoRootDirectory(dirname) { diff --git a/packages/ingest/README.md b/packages/ingest/README.md index 85e685b0f8a..a1ba7a25751 100644 --- a/packages/ingest/README.md +++ b/packages/ingest/README.md @@ -16,6 +16,16 @@ Cumulus is a cloud-based data ingest, archive, distribution and management proto npm install @cumulus/ingest ``` +## Testing + +Running tests locally requires [localstack](https://github.com/localstack/localstack). + +With localstack running, you can run tests using: + +``` +LOCALSTACK_HOST=localhost npm test +``` + ## Modules All modules are accessible using require: `require('@cumulus/ingest/')` or import: `import from '@cumulus/ingest/'`. diff --git a/packages/ingest/aws.js b/packages/ingest/aws.js index 95affd51c15..58cc77beda9 100644 --- a/packages/ingest/aws.js +++ b/packages/ingest/aws.js @@ -7,6 +7,7 @@ const AWS = require('aws-sdk'); const moment = require('moment'); const log = require('@cumulus/common/log'); const errors = require('@cumulus/common/errors'); +const { inTestMode } = require('@cumulus/common/test-utils'); /** * getEndpoint returns proper AWS arguments for various @@ -67,7 +68,7 @@ function getExecutionUrl(executionArn) { } async function invoke(name, payload, type = 'Event') { - if (process.env.IS_LOCAL || process.env.TEST) { + if (process.env.IS_LOCAL || inTestMode()) { log.info(`Faking Lambda invocation for ${name}`); return false; } @@ -98,7 +99,7 @@ function sqs(local) { class Events { static async putEvent(name, schedule, state, description = null, role = null) { - const cwevents = new aws.cloudwatchevents(); + const cwevents = new AWS.CloudWatchEvents(); const params = { Name: name, @@ -213,14 +214,12 @@ class S3 { } static async get(bucket, key) { - const s3 = new AWS.S3(); - const params = { Bucket: bucket, Key: key }; - return s3.getObject(params).promise(); + return aws.s3().getObject(params).promise(); } static async upload(bucket, key, body, acl = 'private') { diff --git a/packages/ingest/crypto.js b/packages/ingest/crypto.js index b11cda060a5..fdadb835755 100644 --- a/packages/ingest/crypto.js +++ b/packages/ingest/crypto.js @@ -33,15 +33,15 @@ class S3KeyPairProvider { } /** - * Decrypt the given string using the given private key stored in the internal bucket + * Decrypt the given string using a private key stored in S3 * - * @param {string} str - The string to encrypt - * @param {string} keyId - The name of the public key to use for encryption - * @param {string} bucket - the optional bucket name. if not provided will - * use env variable "internal" - * @param {stack} stack - the optional stack name. if not provided will - * use env variable "stackName" - * @returns {Promise} the encrypted string + * @param {string} str - The string to decrypt + * @param {string} keyId - The name of the public key to use for decryption + * @param {string} bucket - the optional bucket name. Defaults to the value of + * the "internal" environment variable + * @param {string} stack - the optional stack name. Defaults to the value of + * the "stackName" environment variable + * @returns {Promise.} the decrypted string */ static async decrypt(str, keyId = 'private.pem', bucket = null, stack = null) { const pki = forge.pki; diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 3bc7cfd10d9..3f96ad56acd 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,9 +1,9 @@ { "name": "@cumulus/ingest", - "version": "1.1.4", + "version": "1.2.0", "description": "Ingest utilities", "scripts": { - "test": "env TEST=true ava" + "test": "ava" }, "publishConfig": { "access": "public" @@ -38,9 +38,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", - "@cumulus/pvl": "^1.0.0", - "@cumulus/test-data": "^1.1.3", + "@cumulus/common": "^1.2.0", + "@cumulus/pvl": "^1.2.0", + "@cumulus/test-data": "^1.2.0", "aws-sdk": "^2.4.11", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", @@ -57,7 +57,7 @@ "lodash": "^4.17.5", "lodash.get": "^4.4.2", "mkdirp": "^0.5.1", - "moment": "^2.6.0", + "moment": "^2.21.0", "node-forge": "^0.7.1", "pino": "^4.7.1", "pump": "^1.0.2", diff --git a/packages/ingest/parse-pdr.js b/packages/ingest/parse-pdr.js index 07e5c0acf37..53b70951443 100644 --- a/packages/ingest/parse-pdr.js +++ b/packages/ingest/parse-pdr.js @@ -5,7 +5,7 @@ 'use strict'; -const fs = require('fs'); +const fs = require('fs-extra'); const pvl = require('@cumulus/pvl/t'); const { PDRParsingError } = require('@cumulus/common/errors'); @@ -83,9 +83,12 @@ function extractGranuleId(fileName, regex) { return fileName; } -module.exports.parsePdr = function parsePdr(pdrFilePath, collection, pdrName) { +module.exports.parsePdr = async function parsePdr(pdrFilePath, collection, pdrName) { // then read the file and and pass it to parser - const pdrFile = fs.readFileSync(pdrFilePath, 'utf8'); + const pdrFile = await fs.readFile(pdrFilePath, 'utf8'); + + // Investigating CUMULUS-423 + if (pdrFile.trim().length === 0) throw new Error(`PDR file had no contents: ${pdrFilePath}`); const obj = { granules: [] diff --git a/packages/ingest/pdr.js b/packages/ingest/pdr.js index 0d93dd36288..97481633096 100644 --- a/packages/ingest/pdr.js +++ b/packages/ingest/pdr.js @@ -179,10 +179,10 @@ class Parse { * @return {Promise} * @public */ - parse(pdrLocalPath) { + async parse(pdrLocalPath) { // catching all parse errors here to mark the pdr as failed // if any error occured - const parsed = parsePdr(pdrLocalPath, this.collection, this.pdr.name); + const parsed = await parsePdr(pdrLocalPath, this.collection, this.pdr.name); // each group represents a Granule record. // After adding all the files in the group to the Queue diff --git a/packages/ingest/queue.js b/packages/ingest/queue.js index 702f2789d48..7e5790a1823 100644 --- a/packages/ingest/queue.js +++ b/packages/ingest/queue.js @@ -5,8 +5,6 @@ const { sendSQSMessage, parseS3Uri } = require('@cumulus/common/aws'); -const get = require('lodash.get'); -const uuidv4 = require('uuid/v4'); /** * Create a message from a template stored on S3 @@ -37,7 +35,8 @@ async function enqueueParsePdrMessage( queueUrl, parsePdrMessageTemplateUri, provider, - collection) { + collection +) { const message = await getMessageFromTemplate(parsePdrMessageTemplateUri); message.meta.provider = provider; @@ -87,27 +86,3 @@ async function enqueueGranuleIngestMessage( return sendSQSMessage(queueUrl, message); } exports.enqueueGranuleIngestMessage = enqueueGranuleIngestMessage; - -/** - * Queue a workflow to be picked up by SF starter - * - * @param {Object} event - event to queue with workflow and payload info - * @returns {Promise} - resolves when the message has been enqueued - */ -async function queueWorkflowMessage(event) { - const template = get(event, 'template'); - const provider = get(event, 'provider', {}); - const collection = get(event, 'collection', {}); - const payload = get(event, 'payload', {}); - - const message = await getMessageFromTemplate(template); - - message.meta.provider = provider; - message.meta.collection = collection; - - message.payload = payload; - message.cumulus_meta.execution_name = uuidv4(); - - return sendSQSMessage(message.meta.queues.startSF, message); -} -exports.queueWorkflowMessage = queueWorkflowMessage; diff --git a/packages/ingest/test/queue.js b/packages/ingest/test/queue.js index 3e5c089b15f..0d1db238e6c 100644 --- a/packages/ingest/test/queue.js +++ b/packages/ingest/test/queue.js @@ -2,10 +2,10 @@ const test = require('ava'); const queue = require('../queue'); -const { sqs, s3, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); -const { createQueue, randomString } = require('@cumulus/common/test-utils'); +const { createQueue, sqs, s3, recursivelyDeleteS3Bucket } = require('@cumulus/common/aws'); +const { randomString } = require('@cumulus/common/test-utils'); -test.beforeEach(async (t) => { +test.beforeEach(async(t) => { t.context.templateBucket = randomString(); await s3().createBucket({ Bucket: t.context.templateBucket }).promise(); @@ -19,8 +19,9 @@ test.beforeEach(async (t) => { }, meta: { queues: { startSF: t.context.queueUrl } } }; - + const messageTemplateKey = `${randomString()}/template.json`; + t.context.messageTemplateKey = messageTemplateKey; await s3().putObject({ Bucket: t.context.templateBucket, Key: messageTemplateKey, @@ -30,34 +31,42 @@ test.beforeEach(async (t) => { t.context.template = `s3://${t.context.templateBucket}/${messageTemplateKey}`; }); -test.afterEach(async (t) => { +test.afterEach(async(t) => { await Promise.all([ recursivelyDeleteS3Bucket(t.context.templateBucket), sqs().deleteQueue({ QueueUrl: t.context.queueUrl }).promise() ]); }); -test('the queue receives a correctly formatted workflow message', async (t) => { - const event = { - template: t.context.template, - provider: 'PROV1', - collection: { name: 'test-collection' }, - payload: { test: 'test payload' } - }; +test('the queue receives a correctly formatted workflow message', async(t) => { + const granule = { granuleId: '1', files: [] }; + const { queueUrl } = t.context; + const templateUri = `s3://${t.context.templateBucket}/${t.context.messageTemplateKey}`; + const collection = { name: 'test-collection', version: '0.0.0' }; + const provider = { id: 'test-provider' }; - await queue.queueWorkflowMessage(event); + await queue.enqueueGranuleIngestMessage(granule, queueUrl, templateUri, provider, collection); await sqs().receiveMessage({ QueueUrl: t.context.queueUrl, MaxNumberOfMessages: 10, WaitTimeSeconds: 1 }).promise() - .then((receiveMessageResponse) => { - t.is(receiveMessageResponse.Messages.length, 1); - - const message = JSON.parse(receiveMessageResponse.Messages[0].Body); - t.is(message.meta.provider, 'PROV1'); - t.is(JSON.stringify(message.meta.collection), JSON.stringify({ name: 'test-collection' })); - t.is(JSON.stringify(message.payload), JSON.stringify({ test: 'test payload' })); - t.is(message.cumulus_meta.state_machine, t.context.stateMachineArn); - }); + .then((receiveMessageResponse) => { + t.is(receiveMessageResponse.Messages.length, 1); + + const actualMessage = JSON.parse(receiveMessageResponse.Messages[0].Body); + const expectedMessage = { + cumulus_meta: { + state_machine: t.context.stateMachineArn + }, + meta: { + queues: { startSF: t.context.queueUrl }, + provider: provider, + collection: collection + }, + payload: { granules: [granule] } + }; + + t.deepEqual(expectedMessage, actualMessage); + }); }); diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 354d907a003..96ed89d8f91 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/integration-tests", - "version": "1.1.4", + "version": "1.2.0", "description": "Integration tests", "bin": { "cumulus-test": "./bin/cli.js" @@ -29,7 +29,7 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "@cumulus/deployment": "^1.1.4", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/packages/pvl/package.json b/packages/pvl/package.json index e86c502def3..73a2bd0c722 100644 --- a/packages/pvl/package.json +++ b/packages/pvl/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pvl", - "version": "1.0.0", + "version": "1.2.0", "description": "Parse and serialize Parameter Value Language, a data markup language used by NASA", "main": "index.js", "scripts": { @@ -18,6 +18,6 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { - "ava": "^0.17.0" + "ava": "^0.23.0" } } diff --git a/packages/task-debug/package.json b/packages/task-debug/package.json index 381be64f3ef..981914d126d 100644 --- a/packages/task-debug/package.json +++ b/packages/task-debug/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/task-debug", "private": true, - "version": "1.1.3", + "version": "1.2.0", "description": "A harness for debugging workflows.", "main": "index.js", "repository": { @@ -18,11 +18,11 @@ "test": "test" }, "dependencies": { - "@cumulus/common": "^1.1.3", + "@cumulus/common": "^1.2.0", "commander": "^2.11.0" }, "devDependencies": { - "ava": "^0.21.0", + "ava": "^0.23.0", "babel-cli": "^6.26.0", "babel-core": "^6.25.0", "babel-preset-env": "^1.6.0", diff --git a/packages/test-data/cumulus_messages/pdr-status-check.json b/packages/test-data/cumulus_messages/pdr-status-check.json index fa0f578d9ea..55fdb8c90be 100644 --- a/packages/test-data/cumulus_messages/pdr-status-check.json +++ b/packages/test-data/cumulus_messages/pdr-status-check.json @@ -85,7 +85,11 @@ "running": [ "arn:aws:states:us-east-1:000000000000:execution:LpdaacCumulusIngestGranuleS-pOyNXh5jeR4h:d5b6344a-36eb-4c97-a5cf-3f6e83f0692a" ], - "limit": 30 + "limit": 30, + "pdr": { + "name": "MOD09GQ_1granule_v2.PDR", + "path": "/" + } }, "exception": "None", "workflow_config": { diff --git a/packages/test-data/package.json b/packages/test-data/package.json index 026fa3dc80f..40f30b80593 100644 --- a/packages/test-data/package.json +++ b/packages/test-data/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/test-data", - "version": "1.1.3", + "version": "1.2.0", "description": "Includes the test data for various packages", "keywords": [ "GIBS", diff --git a/tests/ftp_pdr_parse_ingest.js b/tests/ftp_pdr_parse_ingest.js index 63397eeecee..b7f9b31f771 100644 --- a/tests/ftp_pdr_parse_ingest.js +++ b/tests/ftp_pdr_parse_ingest.js @@ -11,8 +11,9 @@ const { deleteCMAFromTasks, messageBuilder } = require('../packages/integration-tests/local'); -const { randomString, createQueue } = require('../packages/common/test-utils'); +const { randomString } = require('../packages/common/test-utils'); const { + createQueue, recursivelyDeleteS3Bucket, s3, sqs, diff --git a/tests/sftp_pdr_parse_ingest.js b/tests/sftp_pdr_parse_ingest.js index 8cc4c3b1fb4..eccac52b338 100644 --- a/tests/sftp_pdr_parse_ingest.js +++ b/tests/sftp_pdr_parse_ingest.js @@ -1,6 +1,5 @@ 'use strict'; -const path = require('path'); const test = require('ava'); const fs = require('fs-extra'); const { @@ -10,8 +9,9 @@ const { deleteCMAFromTasks, messageBuilder } = require('../packages/integration-tests/local'); -const { randomString, createQueue } = require('../packages/common/test-utils'); +const { randomString } = require('../packages/common/test-utils'); const { + createQueue, recursivelyDeleteS3Bucket, s3, sqs, From 24dd67a04484b24af522bb19172eb0e4818e5482 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 Mar 2018 14:21:13 -0400 Subject: [PATCH 2/6] Fix missed conflict --- cumulus/tasks/sf-sns-report/package.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cumulus/tasks/sf-sns-report/package.json b/cumulus/tasks/sf-sns-report/package.json index 36968e4dec0..a7c6bdb2e09 100644 --- a/cumulus/tasks/sf-sns-report/package.json +++ b/cumulus/tasks/sf-sns-report/package.json @@ -38,15 +38,9 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "dependencies": { -<<<<<<< HEAD "@cumulus/common": "^1.2.0", "@cumulus/cumulus-message-adapter-js": "^1.0.1", "@cumulus/ingest": "^1.2.0", -======= - "@cumulus/common": "^1.1.0", - "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.1", ->>>>>>> master "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", From 2427bff85466587753a787a547de18ae6ed7f6e1 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 Mar 2018 14:27:04 -0400 Subject: [PATCH 3/6] Remove duplicate function --- packages/common/aws.js | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/packages/common/aws.js b/packages/common/aws.js index 2b2229bc45a..08ef45fcd2f 100644 --- a/packages/common/aws.js +++ b/packages/common/aws.js @@ -509,34 +509,6 @@ async function createQueue(queueName) { } exports.createQueue = createQueue; -/** - * Create an SQS Queue. Properly handles localstack queue URLs - * - * @param {string} queueName - defaults to a random string - * @returns {Promise.} the Queue URL - */ -async function createQueue(queueName) { - const actualQueueName = queueName || randomString(); - - const createQueueResponse = await exports.sqs().createQueue({ - QueueName: actualQueueName - }).promise(); - - if (inTestMode()) { - // Properly set the Queue URL. This is needed because LocalStack always - // returns the QueueUrl as "localhost", even if that is not where it should - // actually be found. CircleCI breaks without this. - const returnedQueueUrl = url.parse(createQueueResponse.QueueUrl); - returnedQueueUrl.host = undefined; - returnedQueueUrl.hostname = process.env.LOCALSTACK_HOST; - - return url.format(returnedQueueUrl); - } - - return createQueueResponse.QueueUrl; -} -exports.createQueue = createQueue; - /** * Send a message to AWS SQS * From 96b7f80b6d0fdd55ce138c05f33d9c3b65f1a267 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 Mar 2018 14:30:56 -0400 Subject: [PATCH 4/6] Remove 260 from changelog 1.1.2 section --- CHANGELOG.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba7c5ab3329..070a1566bdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,11 +50,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `@cumulus/deployment` deploys DynamoDB streams for the Collections, Providers and Rules tables as well as a new lambda function called `dbIndexer`. The `dbIndexer` lambda has an event source mapping which listens to each of the DynamoDB streams. The dbIndexer lambda receives events referencing operations on the DynamoDB table and updates the elasticsearch cluster accordingly. - The `@cumulus/api` endpoints for collections, providers and rules _only_ query DynamoDB, with the exception of LIST endpoints and the collections' GET endpoint. -- **CUMULUS-260: "PDR page on dashboard only shows zeros."** The PDR stats in LPDAAC are all 0s, even if the dashboard has been fixed to retrieve the correct fields. The current version of pdr-status-check has a few issues. - - pdr is not included in the input/output schema. It's available from the input event. So the pdr status and stats are not updated when the ParsePdr workflow is complete. Adding the pdr to the input/output of the task will fix this. - - pdr-status-check doesn't update pdr stats which prevent the real time pdr progress from showing up in the dashboard. To solve this, added lambda function sf-sns-report which is copied from @cumulus/api/lambdas/sf-sns-broadcast with modification, sf-sns-report can be used to report step function status anywhere inside a step function. So add step sf-sns-report after each pdr-status-check, we will get the PDR status progress at real time. - - It's possible an execution is still in the queue and doesn't exist in sfn yet. Added code to handle 'ExecutionDoesNotExist' error when checking the execution status. - ### Updated - Broke up `kes.override.js` of @cumulus/deployment to multiple modules and moved to a new location - Expanded @cumulus/deployment test coverage From 730df546d4eae6693a15559ebdd83a3d1b8f9424 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 Mar 2018 14:35:19 -0400 Subject: [PATCH 5/6] Fix merge resolution error --- packages/api/endpoints/collections.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/endpoints/collections.js b/packages/api/endpoints/collections.js index a7db56353ac..f676aa0e705 100644 --- a/packages/api/endpoints/collections.js +++ b/packages/api/endpoints/collections.js @@ -123,7 +123,6 @@ function handler(event, context) { return context.fail('HttpMethod is missing'); } - return handle(event, context, !inTestMode() /* authCheck */, cb => { return handle(event, context, !inTestMode() /* authCheck */, (cb) => { if (event.httpMethod === 'GET' && event.pathParameters) { get(event, cb); From 1833899309df2a9494ec983d5fb708e7d53cfcdc Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 Mar 2018 14:49:48 -0400 Subject: [PATCH 6/6] Fix high water mark --- .eslint-ratchet-high-water-mark | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslint-ratchet-high-water-mark b/.eslint-ratchet-high-water-mark index 239cbd5e348..340b3519a5d 100644 --- a/.eslint-ratchet-high-water-mark +++ b/.eslint-ratchet-high-water-mark @@ -1 +1 @@ -1577 +1576