From 089a9a83c5be93b9140736d3480255be311c128a Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Wed, 14 Mar 2018 13:38:02 -0400 Subject: [PATCH 1/5] Starting to add uselist to parsepdr --- cumulus/tasks/parse-pdr/index.js | 3 ++- cumulus/tasks/parse-pdr/schemas/config.json | 5 +++++ packages/ingest/pdr.js | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cumulus/tasks/parse-pdr/index.js b/cumulus/tasks/parse-pdr/index.js index 8aa992b8cfc..b4987abb48a 100644 --- a/cumulus/tasks/parse-pdr/index.js +++ b/cumulus/tasks/parse-pdr/index.js @@ -32,7 +32,8 @@ function parsePdr(event) { config.stack, config.bucket, config.collection, - provider + provider, + config.useList ); return parse.ingest() diff --git a/cumulus/tasks/parse-pdr/schemas/config.json b/cumulus/tasks/parse-pdr/schemas/config.json index 23b94ab5af7..bc88e837665 100644 --- a/cumulus/tasks/parse-pdr/schemas/config.json +++ b/cumulus/tasks/parse-pdr/schemas/config.json @@ -38,6 +38,11 @@ "name": { "type": "string" }, "granuleIdExtraction": { "type": "string" } } + }, + "useList": { + "description": "flag to tell ftp server to use 'LIST' instead of 'STAT'", + "default": false, + "type": "boolean" } } } diff --git a/packages/ingest/pdr.js b/packages/ingest/pdr.js index afe3d5b747b..0d93dd36288 100644 --- a/packages/ingest/pdr.js +++ b/packages/ingest/pdr.js @@ -106,6 +106,7 @@ class Parse { bucket, collection, provider, + useList = false, folder = 'pdrs') { if (this.constructor === Parse) { throw new TypeError('Can not construct abstract class.'); @@ -117,6 +118,7 @@ class Parse { this.collection = collection; this.provider = provider; this.folder = folder; + this.useList = useList; this.port = get(this.provider, 'port', 21); this.host = get(this.provider, 'host', null); From bc651f6cbb3e9bbccb8940c88eff89ae7da8286d Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Wed, 14 Mar 2018 15:41:48 -0400 Subject: [PATCH 2/5] Adding useList to parsePDR config validation --- cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js index a15399750cf..0cb3ff3d98d 100644 --- a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js +++ b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js @@ -32,6 +32,7 @@ test('parse PDR from FTP endpoint', async (t) => { username: 'testuser', password: 'testpass' }; + newPayload.config.useList = true; await validateConfig(t, newPayload.config); From 7102a499ff3e310f99371707a142769f2f5ba1c6 Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 08:54:42 -0400 Subject: [PATCH 3/5] updated CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07b23e783bb..bdfe1a06aa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - added tools to @cumulus/integration-tests for local integration testing - added end to end testing for discovering and parsing of PDRs - `yarn e2e` command is available for end to end testing +- added flag `useList` to parse-pdr ### Fixed - **CUMULUS-326: "Occasionally encounter "Too Many Requests" on deployment"** The api gateway calls will handle throttling errors From 2911f4becb126667365253ee707fcc891554615a Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 09:25:54 -0400 Subject: [PATCH 4/5] Moved added line --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0021194ce2d..b6afe740a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added +- added flag `useList` to parse-pdr + ## [v1.1.3] - 2018-03-14 ### Fixed - Changed @cumulus/deployment package install behavior. The build process will happen after installation @@ -16,7 +19,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - added tools to @cumulus/integration-tests for local integration testing - added end to end testing for discovering and parsing of PDRs - `yarn e2e` command is available for end to end testing -- added flag `useList` to parse-pdr ### Fixed - **CUMULUS-326: "Occasionally encounter "Too Many Requests" on deployment"** The api gateway calls will handle throttling errors From 2738bf6304f9dae33ac58b449f49bc3c4a729dd6 Mon Sep 17 00:00:00 2001 From: Katherine Kelly Date: Thu, 15 Mar 2018 09:29:00 -0400 Subject: [PATCH 5/5] added ticket number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6afe740a27..be33ce246d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added -- added flag `useList` to parse-pdr +- added flag `useList` to parse-pdr [CUMULUS-404] ## [v1.1.3] - 2018-03-14 ### Fixed