CUMULUS-404 Add useList to parsePDR

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - added tools to @cumulus/integration-tests for local integration testing
 - added end to end testing for discovering and parsing of PDRs
 - `yarn e2e` command is available for end to end testing
+- added flag `useList` to parse-pdr
 ### Fixed
 
 - **CUMULUS-326: "Occasionally encounter "Too Many Requests" on deployment"** The api gateway calls will handle throttling errors

cumulus/tasks/parse-pdr/index.js

@@ -32,7 +32,8 @@ function parsePdr(event) {
     config.stack,
     config.bucket,
     config.collection,
-    provider
+    provider,
+    config.useList
   );
 
   return parse.ingest()

cumulus/tasks/parse-pdr/schemas/config.json

@@ -38,6 +38,11 @@
         "name": { "type": "string" },
         "granuleIdExtraction": { "type": "string" }
       }
+    },
+    "useList": {
+      "description": "flag to tell ftp server to use 'LIST' instead of 'STAT'",
+      "default": false,
+      "type": "boolean"
     }
   }
 }

cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js

@@ -32,6 +32,7 @@ test('parse PDR from FTP endpoint', async (t) => {
     username: 'testuser',
     password: 'testpass'
   };
+  newPayload.config.useList = true;
 
   await validateConfig(t, newPayload.config);
 

packages/ingest/pdr.js

@@ -106,6 +106,7 @@ class Parse {
     bucket,
     collection,
     provider,
+    useList = false,
     folder = 'pdrs') {
     if (this.constructor === Parse) {
       throw new TypeError('Can not construct abstract class.');
@@ -117,6 +118,7 @@ class Parse {
     this.collection = collection;
     this.provider = provider;
     this.folder = folder;
+    this.useList = useList;
 
     this.port = get(this.provider, 'port', 21);
     this.host = get(this.provider, 'host', null);