From 089a9a83c5be93b9140736d3480255be311c128a Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Wed, 14 Mar 2018 13:38:02 -0400 Subject: [PATCH 1/9] Starting to add uselist to parsepdr --- cumulus/tasks/parse-pdr/index.js | 3 ++- cumulus/tasks/parse-pdr/schemas/config.json | 5 +++++ packages/ingest/pdr.js | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cumulus/tasks/parse-pdr/index.js b/cumulus/tasks/parse-pdr/index.js index 8aa992b8cfc..b4987abb48a 100644 --- a/cumulus/tasks/parse-pdr/index.js +++ b/cumulus/tasks/parse-pdr/index.js @@ -32,7 +32,8 @@ function parsePdr(event) { config.stack, config.bucket, config.collection, - provider + provider, + config.useList ); return parse.ingest() diff --git a/cumulus/tasks/parse-pdr/schemas/config.json b/cumulus/tasks/parse-pdr/schemas/config.json index 23b94ab5af7..bc88e837665 100644 --- a/cumulus/tasks/parse-pdr/schemas/config.json +++ b/cumulus/tasks/parse-pdr/schemas/config.json @@ -38,6 +38,11 @@ "name": { "type": "string" }, "granuleIdExtraction": { "type": "string" } } + }, + "useList": { + "description": "flag to tell ftp server to use 'LIST' instead of 'STAT'", + "default": false, + "type": "boolean" } } } diff --git a/packages/ingest/pdr.js b/packages/ingest/pdr.js index afe3d5b747b..0d93dd36288 100644 --- a/packages/ingest/pdr.js +++ b/packages/ingest/pdr.js @@ -106,6 +106,7 @@ class Parse { bucket, collection, provider, + useList = false, folder = 'pdrs') { if (this.constructor === Parse) { throw new TypeError('Can not construct abstract class.'); @@ -117,6 +118,7 @@ class Parse { this.collection = collection; this.provider = provider; this.folder = folder; + this.useList = useList; this.port = get(this.provider, 'port', 21); this.host = get(this.provider, 'host', null); From bc651f6cbb3e9bbccb8940c88eff89ae7da8286d Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Wed, 14 Mar 2018 15:41:48 -0400 Subject: [PATCH 2/9] Adding useList to parsePDR config validation --- cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js index a15399750cf..0cb3ff3d98d 100644 --- a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -32,6 +32,7 @@ test('parse PDR from FTP endpoint', async (t) => { username: 'testuser', password: 'testpass' }; + newPayload.config.useList = true; await validateConfig(t, newPayload.config); From 7102a499ff3e310f99371707a142769f2f5ba1c6 Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 08:54:42 -0400 Subject: [PATCH 3/9] updated CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07b23e783bb..bdfe1a06aa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - added tools to @cumulus/integration-tests for local integration testing - added end to end testing for discovering and parsing of PDRs - `yarn e2e` command is available for end to end testing +- added flag `useList` to parse-pdr ### Fixed - **CUMULUS-326: "Occasionally encounter "Too Many Requests" on deployment"** The api gateway calls will handle throttling errors From 2911f4becb126667365253ee707fcc891554615a Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 09:25:54 -0400 Subject: [PATCH 4/9] Moved added line --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0021194ce2d..b6afe740a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added +- added flag `useList` to parse-pdr + ## [v1.1.3] - 2018-03-14 ### Fixed - Changed @cumulus/deployment package install behavior. The build process will happen after installation @@ -16,7 +19,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - added tools to @cumulus/integration-tests for local integration testing - added end to end testing for discovering and parsing of PDRs - `yarn e2e` command is available for end to end testing -- added flag `useList` to parse-pdr ### Fixed - **CUMULUS-326: "Occasionally encounter "Too Many Requests" on deployment"** The api gateway calls will handle throttling errors From 2738bf6304f9dae33ac58b449f49bc3c4a729dd6 Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 09:29:00 -0400 Subject: [PATCH 5/9] added ticket number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6afe740a27..be33ce246d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added -- added flag `useList` to parse-pdr +- added flag `useList` to parse-pdr [CUMULUS-404] ## [v1.1.3] - 2018-03-14 ### Fixed From 784f98e7a46716eda6c6a4a2e90bc4a039dfd350 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 15 Mar 2018 10:36:48 -0400 Subject: [PATCH 6/9] pass encrypted password to api granule --- CHANGELOG.md | 3 +++ packages/deployment/app/cloudformation.template.yml | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be33ce246d1..d59c78e63b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - added flag `useList` to parse-pdr [CUMULUS-404] +### Fixed +- Pass encrypted password to the ApiGranule Lambda function [CUMULUS-424] + ## [v1.1.3] - 2018-03-14 ### Fixed - Changed @cumulus/deployment package install behavior. The build process will happen after installation diff --git a/packages/deployment/app/cloudformation.template.yml b/packages/deployment/app/cloudformation.template.yml index 68e8c3700e4..24c4ab5d8b9 100644 --- a/packages/deployment/app/cloudformation.template.yml +++ b/packages/deployment/app/cloudformation.template.yml @@ -27,7 +27,9 @@ Resources: Properties: Parameters: CmrPassword: - Ref: CmrPassword + Fn::GetAtt: + - CumulusCustomResource + - CmrPassword ElasticSearchDomain: Fn::GetAtt: - {{../es.name}}Domain From 804eb3a57647b1518e6ba72dca71a3e8fc4e90fb Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 10:58:13 -0400 Subject: [PATCH 7/9] Updating version to 1.1.4 --- CHANGELOG.md | 1 + cumulus/tasks/discover-granules/package.json | 4 ++-- cumulus/tasks/discover-pdr/package.json | 4 ++-- cumulus/tasks/discover-pdrs/package.json | 4 ++-- cumulus/tasks/discover-s3-granules/package.json | 4 ++-- cumulus/tasks/generate-pdr-file-list/package.json | 4 ++-- cumulus/tasks/parse-pdr/package.json | 4 ++-- cumulus/tasks/pdr-status-check/package.json | 4 ++-- cumulus/tasks/post-to-cmr/package.json | 4 ++-- cumulus/tasks/queue-granules/package.json | 4 ++-- cumulus/tasks/queue-pdrs/package.json | 4 ++-- cumulus/tasks/sync-granule/package.json | 4 ++-- cumulus/tasks/validate-pdr/package.json | 4 ++-- lerna.json | 2 +- packages/api/package.json | 4 ++-- packages/deployment/package.json | 2 +- packages/ingest/package.json | 2 +- packages/integration-tests/package.json | 4 ++-- 18 files changed, 32 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d59c78e63b9..ed4de491257 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v1.1.4] - 2018-03-15 ### Added - added flag `useList` to parse-pdr [CUMULUS-404] diff --git a/cumulus/tasks/discover-granules/package.json b/cumulus/tasks/discover-granules/package.json index bfcaaaaf987..435fd708cac 100644 --- a/cumulus/tasks/discover-granules/package.json +++ b/cumulus/tasks/discover-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-granules", - "version": "1.1.3", + "version": "1.1.4", "description": "Discover Granules in FTP/HTTP/SFTP endpoints", "main": "index.js", "directories": { @@ -39,7 +39,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/discover-pdr/package.json b/cumulus/tasks/discover-pdr/package.json index 949771f38f2..a49ae6aaf79 100644 --- a/cumulus/tasks/discover-pdr/package.json +++ b/cumulus/tasks/discover-pdr/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/discover-pdr", "private": true, - "version": "1.1.3", + "version": "1.1.4", "description": "SIPS handler PDR discovery task", "main": "index.js", "keywords": [ @@ -37,7 +37,7 @@ }, "dependencies": { "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "ftp": "^0.3.10", "node-fetch": "^1.6.1", "parse-duration": "^0.1.1", diff --git a/cumulus/tasks/discover-pdrs/package.json b/cumulus/tasks/discover-pdrs/package.json index b1eacbb81da..4d493ea8c03 100644 --- a/cumulus/tasks/discover-pdrs/package.json +++ b/cumulus/tasks/discover-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-pdrs", - "version": "1.1.3", + "version": "1.1.4", "description": "Discover PDRs in FTP and HTTP endpoints", "main": "index.js", "directories": { @@ -38,7 +38,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/discover-s3-granules/package.json b/cumulus/tasks/discover-s3-granules/package.json index 5ed7e9242a9..a70618d4523 100644 --- a/cumulus/tasks/discover-s3-granules/package.json +++ b/cumulus/tasks/discover-s3-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/discover-s3-granules", - "version": "1.1.3", + "version": "1.1.4", "description": "Discover granules from an S3 bucket", "main": "index.js", "directories": { @@ -37,7 +37,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/generate-pdr-file-list/package.json b/cumulus/tasks/generate-pdr-file-list/package.json index 06b6c922d9a..502adc19079 100644 --- a/cumulus/tasks/generate-pdr-file-list/package.json +++ b/cumulus/tasks/generate-pdr-file-list/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/generate-pdr-file-list", "private": true, - "version": "1.1.3", + "version": "1.1.4", "description": "SIPS handler PDR processing task", "main": "index.js", "keywords": [ @@ -36,7 +36,7 @@ }, "dependencies": { "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/pvl": "^1.0.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", diff --git a/cumulus/tasks/parse-pdr/package.json b/cumulus/tasks/parse-pdr/package.json index 654fa73b2fc..368b45466a9 100644 --- a/cumulus/tasks/parse-pdr/package.json +++ b/cumulus/tasks/parse-pdr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/parse-pdr", - "version": "1.1.3", + "version": "1.1.4", "description": "Download and Parse a given PDR", "main": "index.js", "directories": { @@ -41,7 +41,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/pdr-status-check/package.json b/cumulus/tasks/pdr-status-check/package.json index 61135fba05e..25ef23a59b9 100644 --- a/cumulus/tasks/pdr-status-check/package.json +++ b/cumulus/tasks/pdr-status-check/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/pdr-status-check", - "version": "1.1.3", + "version": "1.1.4", "description": "Checks execution status of granules in a PDR", "main": "index.js", "directories": { @@ -42,7 +42,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/post-to-cmr/package.json b/cumulus/tasks/post-to-cmr/package.json index 14e3415a9a2..a0cbffabf0f 100644 --- a/cumulus/tasks/post-to-cmr/package.json +++ b/cumulus/tasks/post-to-cmr/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/post-to-cmr", - "version": "1.1.3", + "version": "1.1.4", "description": "Post a given granule to CMR", "main": "index.js", "directories": { @@ -44,7 +44,7 @@ "@cumulus/cmrjs": "^1.1.3", "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/queue-granules/package.json b/cumulus/tasks/queue-granules/package.json index f45c0375269..497993a3507 100644 --- a/cumulus/tasks/queue-granules/package.json +++ b/cumulus/tasks/queue-granules/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-granules", - "version": "1.1.3", + "version": "1.1.4", "description": "Add discovered granules to the queue", "main": "index.js", "directories": { @@ -40,7 +40,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", diff --git a/cumulus/tasks/queue-pdrs/package.json b/cumulus/tasks/queue-pdrs/package.json index e92e2e1dc8c..f3f478f5cd3 100644 --- a/cumulus/tasks/queue-pdrs/package.json +++ b/cumulus/tasks/queue-pdrs/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/queue-pdrs", - "version": "1.1.3", + "version": "1.1.4", "description": "Add discovered PDRs to a queue", "main": "index.js", "directories": { @@ -39,7 +39,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", diff --git a/cumulus/tasks/sync-granule/package.json b/cumulus/tasks/sync-granule/package.json index 867531427af..d4f6fc8fb1a 100644 --- a/cumulus/tasks/sync-granule/package.json +++ b/cumulus/tasks/sync-granule/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/sync-granule", - "version": "1.1.3", + "version": "1.1.4", "description": "Download a given granule", "main": "index.js", "directories": { @@ -43,7 +43,7 @@ "dependencies": { "@cumulus/common": "^1.1.3", "@cumulus/cumulus-message-adapter-js": "^1.0.1", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/test-data": "^1.1.3", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", diff --git a/cumulus/tasks/validate-pdr/package.json b/cumulus/tasks/validate-pdr/package.json index 7e1ec2f07e5..194a6f98bca 100644 --- a/cumulus/tasks/validate-pdr/package.json +++ b/cumulus/tasks/validate-pdr/package.json @@ -1,7 +1,7 @@ { "name": "@cumulus/validate-pdr", "private": true, - "version": "1.1.3", + "version": "1.1.4", "description": "SIPS handler PDR processing task", "main": "index.js", "keywords": [ @@ -40,7 +40,7 @@ }, "dependencies": { "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/pvl": "^1.0.0", "ftp": "^0.3.10", "node-fetch": "^1.6.1", diff --git a/lerna.json b/lerna.json index 860e83b27b3..4022c39ff8c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "2.9.0", - "version": "1.1.3", + "version": "1.1.4", "packages": [ "packages/*", "cumulus/tasks/*", diff --git a/packages/api/package.json b/packages/api/package.json index bc9709e293c..0922d2384c9 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/api", - "version": "1.1.3", + "version": "1.1.4", "description": "Lambda functions for handling all daac's API operations", "main": "index.js", "scripts": { @@ -34,7 +34,7 @@ "dependencies": { "@cumulus/cmrjs": "^1.1.3", "@cumulus/common": "^1.1.3", - "@cumulus/ingest": "^1.1.3", + "@cumulus/ingest": "^1.1.4", "@cumulus/pvl": "^1.0.0", "ajv": "^5.2.2", "archiver": "^2.1.1", diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 75f42ad0734..9bba4219745 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/deployment", - "version": "1.1.3", + "version": "1.1.4", "description": "Deployment templates for cumulus", "scripts": { "test": "ava --no-color", diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 8bbbe4946c7..3bc7cfd10d9 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/ingest", - "version": "1.1.3", + "version": "1.1.4", "description": "Ingest utilities", "scripts": { "test": "env TEST=true ava" diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index caf7c235e4d..354d907a003 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -1,6 +1,6 @@ { "name": "@cumulus/integration-tests", - "version": "1.1.3", + "version": "1.1.4", "description": "Integration tests", "bin": { "cumulus-test": "./bin/cli.js" @@ -30,7 +30,7 @@ "license": "Apache-2.0", "dependencies": { "@cumulus/common": "^1.1.3", - "@cumulus/deployment": "^1.1.3", + "@cumulus/deployment": "^1.1.4", "babel-core": "^6.25.0", "babel-loader": "^6.2.4", "babel-plugin-transform-async-to-generator": "^6.24.1", From 39ec01f058640be61180892dda7d731042df00cf Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 10:59:55 -0400 Subject: [PATCH 8/9] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4de491257..d0204128e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v1.0.0] - 2018-02-23 -[Unreleased]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.3...HEAD +[Unreleased]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.4...HEAD +[v1.1.4]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.3...v1.1.4 [v1.1.3]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.2...v1.1.3 [v1.1.2]: https://github.com/cumulus-nasa/cumulus/compare/v1.1.1...v1.1.2 [v1.1.1]: https://github.com/cumulus-nasa/cumulus/compare/v1.0.1...v1.1.1 From 154d255696a39f3a6ac4880ee0fba9026b764361 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 15 Mar 2018 11:41:50 -0400 Subject: [PATCH 9/9] run publish again release branch --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bd676cbea4..9c62bd89125 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -196,4 +196,4 @@ workflows: - build_and_test filters: branches: - only: master \ No newline at end of file + only: release \ No newline at end of file