try old recursion

docs/tasks.md

@@ -123,4 +123,4 @@ Download a given granule
 Fake processing task used for integration tests
 
 - Schemas: See this module's [schema definitions](https://github.com/nasa/cumulus/tree/master/tasks/test-processing/schemas).
-- Resources: [npm](https://npmjs.com/package/@cumulus/test-processing) | [source](https://github.com/nasa/cumulus) | [web](https://github.com/nasa/cumulus/tree/master/tasks/test-processing)
+- Resources: [npm](https://npmjs.com/package/@cumulus/test-processing) | [source](https://github.com/nasa/cumulus) | [web](https://github.com/nasa/cumulus/tree/master/tasks/test-processing)

packages/common/sftp.js

@@ -163,10 +163,9 @@ class Sftp {
    * @returns {Promise.<Object>} - list of file object
    */
   async list(remotePath) {
-    const normalizedPath = (remotePath.trim() === '' ? '/' : remotePath);
     if (!this.connected) await this.connect();
     return new Promise((resolve, reject) => {
-      this.sftp.readdir(normalizedPath, (err, list) => {
+      this.sftp.readdir(remotePath, (err, list) => {
         if (err) {
           if (err.message.includes('No such file')) {
             return resolve([]);
@@ -175,7 +174,7 @@ class Sftp {
         }
         return resolve(list.map((i) => ({
           name: i.filename,
-          path: normalizedPath,
+          path: remotePath,
           type: i.longname.substr(0, 1),
           size: i.attrs.size,
           time: i.attrs.mtime * 1000

packages/ingest/recursion.js

@@ -1,4 +1,4 @@
-'use strict';
+/*'use strict';
 
 const path = require('path');
 const log = require('@cumulus/common/log');
@@ -12,7 +12,7 @@ const log = require('@cumulus/common/log');
  * @param {Array<string>} segments - path segments
  * @param {number} position - current position in the segment list
  * @returns {Array<Object>} - filtered contents of directory
- */
+ *
 async function recurOnDirectory(fn, currentPath, segments, position) {
   // interpret the next path segment as a regex for filtering, and
   // recursively list everything when we've run out of segments
@@ -55,7 +55,7 @@ async function recurOnDirectory(fn, currentPath, segments, position) {
  *                                  regexes for filtering
  * @returns {Promise} the promise of an object that has the path is the key and
  *   list of files as values
- */
+ *
 async function recursion(fn, configuredPath) {
   const normalizedPath = path.normalize(configuredPath);
   const isAbsolutePath = path.isAbsolute(normalizedPath);
@@ -70,6 +70,89 @@ async function recursion(fn, configuredPath) {
     log.info('Falling back to unfiltered directory listing...');
     return recurOnDirectory(fn, normalizedPath, [], 0);
   }
+}*/
+
+'use strict';
+
+const join = require('path').join;
+const log = require('@cumulus/common/log');
+
+/**
+ * Handles recursion of a FTP/SFTP list operation
+ * It requests a promisified list function that returns contents of
+ * a directory on a server
+ *
+ * @param {function} fn The promisified function for listing a directory
+ * @param {string} originalPath the full path to use for recursively listing the directory
+ * @param {string} [currentPath=null] the current directory to list recursively
+ * @param {number} [position=0] current position in the recursive tree
+ * @returns {Promise} the promise of an object that has the path is the key and
+ *   list of files as values
+ */
+async function recursion(fn, originalPath, currentPath = null, position = 0) {
+  // build the recursion path object
+  const regex = /(\(.*?\))/g;
+  const rules = (originalPath || '/').split(regex).map((i) => i.replace(/\\\\/g, '\\'));
+  const map = rules.map((r) => (r.match(regex) !== null));
+
+  let files = [];
+  let path = currentPath;
+  if (!path) {
+    path = rules[position];
+  }
+
+  log.info(`Listing ${path}`);
+
+  // get list of current path
+  const list = await fn(path);
+
+  // loop try what is returned
+  for (let ctr = 0; ctr < list.length; ctr += 1) {
+    const item = list[ctr];
+
+    let regexPath;
+    let textPath;
+    let newPath;
+
+    // if directory is found, apply recursion logic
+    if (item.type === 'd' || item.type === 1) {
+      // first we check if the next segment of the provided path
+      // is a regex rule
+      const isRegex = map[position + 1];
+
+      // if it is the regex rule, we use the rule to
+      // decide whether to do more recursion
+      if (isRegex) {
+        regexPath = new RegExp(rules[position + 1]);
+      } else {
+        // if it is just a path, we use that segment and its children
+        textPath = rules[position + 1];
+      }
+
+      //  if next segment is regex and matches the rule
+      //  list
+      if (isRegex && item.name.match(regexPath)) {
+        newPath = join(path, item.name);
+      } else if (textPath) {
+        // if it is a regular path, use the path
+        newPath = join(path, textPath);
+      } else {
+        // and ignore all cases that does't match this rule
+        continue; // eslint-disable-line no-continue
+      }
+
+      // eslint-disable-next-line no-await-in-loop
+      const tmp = await recursion(fn, originalPath, newPath, position + 1);
+      files = files.concat(tmp);
+
+      if (textPath) break;
+    } else if (item.type === '-' || item.type === 0) {
+      // add file to the list
+      files = files.concat(item);
+    }
+  }
+
+  return files;
 }
 
 module.exports = recursion;

packages/sftp-client/index.js

@@ -134,12 +134,10 @@ class SftpClient {
    * @returns {Promise<Array<Object>>} list of file objects
    */
   async list(remotePath) {
-    const actualPath = remotePath.trim() === '' ? '/' : remotePath;
-
     await this.connect();
 
     return new Promise((resolve, reject) => {
-      this.sftp.readdir(actualPath, (err, list) => {
+      this.sftp.readdir(remotePath, (err, list) => {
         if (err) {
           if (err.message.includes('No such file')) {
             return resolve([]);
@@ -148,7 +146,7 @@ class SftpClient {
         }
         return resolve(list.map((i) => ({
           name: i.filename,
-          path: actualPath,
+          path: remotePath,
           type: i.longname.substr(0, 1),
           size: i.attrs.size,
           time: i.attrs.mtime * 1000