Gitc 567 pre v1

cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml

@@ -50,6 +50,8 @@ _templates:
             #temporal: 2015-01-01T00:00:00Z,2015-01-02T18:39:00Z # Exactly 949 (+1 that has no day_night_flag)
           granule_meta:
             key: '{meta.collection}/{granule.id}'
+            polygons: '{granule.polygons}'
+            box: '{granule.box}'
             granuleId: '{granule.id}'
             version: '{granule.updated}'
             date:
@@ -99,6 +101,36 @@ _templates:
             - Blank_RGBA_512.png
             - mrfgen_configuration_file.xml
 
+          alternate_commands:
+            - gdal_command: gdalwarp
+              args: [
+                -of, GTiff,
+                -cutline, /tmp/left_side_cmr.json,
+                -crop_to_cutline,
+                -t_srs, EPSG:4326,
+                -tr, 0.0001373291015625, 0.0001373291015625,
+                -srcnodata, "0 0 0",
+                -dstalpha,
+                -r, near,
+                -overwrite,
+                input.tif,
+                in/processed_left_4326.tif
+              ]
+            - gdal_command: gdalwarp
+              args: [
+                -of, GTiff,
+                -cutline, /tmp/right_side_cmr.json,
+                -crop_to_cutline,
+                -t_srs, EPSG:4326,
+                -tr, 0.0001373291015625, 0.0001373291015625,
+                -srcnodata, "0 0 0",
+                -dstalpha,
+                -r, near,
+                -overwrite,
+                input.tif,
+                in/processed_right_4326.tif
+              ]
+
           commands:
             - gdal_command: gdalwarp
               args: [

cumulus/tasks/discover-cmr-granules/index.js

@@ -28,7 +28,7 @@ function validateParameter(config, param) {
  * @returns {bool} - true or exception
  */
 function validateParameters(config) {
-  const params = ['root', 'event', 'granule_meta', 'query'];
+  const params = ['root', 'granule_meta', 'query'];
   for (const p of params) validateParameter(config, p);
 
   return true;
@@ -62,16 +62,16 @@ module.exports = class DiscoverCmrGranulesTask extends Task {
     const filtered = this.excludeFiltered(messages, this.config.filtered_granule_keys);
 
     // Write the messages to a DynamoDB table so we can track ingest failures
-    const messagePromises = filtered.map(msg => {
+    const messagePromises = filtered.map((msg) => {
       const { granuleId, version, collection } = msg.meta;
       const params = {
         TableName: this.config.ingest_tracking_table,
         Item: {
           'granule-id': granuleId,
-          'version': version,
-          'collection': collection,
+          version: version,
+          collection: collection,
           'ingest-start-datetime': moment().format(),
-          'message': JSON.stringify(msg)
+          message: JSON.stringify(msg)
         }
       };
       return docClient.put(params).promise();
@@ -116,6 +116,7 @@ module.exports = class DiscoverCmrGranulesTask extends Task {
    *
    * @param {string} root - The CMR root url (protocol and domain without path)
    * @param {Object} query - The query parameters to serialize and send to a CMR granules search
+   * @param {string} scrollID - The scroll id to pass to the CMR harvesting API
    * @returns {Array} An array of all granules matching the given query
    */
   async cmrGranules(root, query, scrollID) {
@@ -195,8 +196,9 @@ module.exports = class DiscoverCmrGranulesTask extends Task {
 
 // To use with Visual Studio Code Debugger, uncomment next block
 //
-//global.__isDebug = true;
-//const local = require('@cumulus/common/local-helpers');
-//const localTaskName = 'DiscoverCmrGranules';
-//local.setupLocalRun(module.exports.handler, local.collectionMessageInput(
-//  'MOPITT_DCOSMR_LL_D_STD', localTaskName));
+// global.__isDebug = true;
+// const local = require('@cumulus/common/local-helpers');
+// const localTaskName = 'DiscoverCmrGranules';
+// local.setupLocalRun(module.exports.handler, local.collectionMessageInput(
+//  'AST_L1T_DAY', localTaskName, (o) => o, `${local.fileRoot()}/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml`));
+

cumulus/tasks/discover-cmr-granules/test/discover-cmr-granules-spec.js

@@ -35,15 +35,6 @@ test('check with invalid root parameter', async (t) => {
   t.is(errors, 'Undefined root parameter');
 });
 
-test('check with invalid event parameter', async (t) => {
-  // Remove event parameter for  DiscoverCmrGranules Task
-  const newPayload = _.cloneDeep(message); //Object assign will not work here
-  delete newPayload.workflow_config_template.DiscoverCmrGranules.event;
-
-  const [errors] = await testHelpers.run(DiscoverCmrGranules, newPayload);
-  t.is(errors, 'Undefined event parameter');
-});
-
 test('check with invalid granule_meta parameter', async (t) => {
   // Remove granule_meta parameter for  DiscoverCmrGranules Task
   const newPayload = _.cloneDeep(message); //Object assign will not work here

cumulus/tasks/run-gdal/index.js

@@ -6,32 +6,89 @@ const log = require('@cumulus/common/log');
 const fs = require('fs');
 const path = require('path');
 const spawn = require('child_process').spawn;
+const rimraf = require('rimraf');
 
 /**
  *
  */
 module.exports = class RunGdalTask extends Task {
+
+  /**
+   * Compute the minimum bounding rectangle (MBR) for the given polygon. Assumes the maximum
+   * longitudinal distance * between points is less than 180 degrees.
+   * NOTE: This will NOT work for polygons covering poles.
+   * @param {string} polyString  A polygon from a CMR metadata response. This has the form
+   * "lat_0 lon_0 lat_1 lon_1 ... lat_n lon_n lat_0 lon_0"
+   * @returns {Array} An array containing the mbr in the form [latBL, lonBL, latUR, lonRU] where
+   * latBL = latitude of the bottom left corner
+   * lonBL = longitude of the bottom left corner
+   * latUR = latitude of the upper right corner
+   * lonUR = longitude of the upper right corner
+   */
+  static computeMbr(polyString) {
+    const coords = polyString.split(' ');
+    let minLat = 360.0;
+    let minLon = 360.0;
+    let maxLat = -360.0;
+    let maxLon = -360.0;
+    for (let i = 1; i < coords.length / 2; i++) {
+      const lat = parseFloat(coords[i * 2]);
+      const lon = parseFloat(coords[(i * 2) + 1]);
+
+      if (lon > maxLon) maxLon = lon;
+      if (lon < minLon) minLon = lon;
+      if (lat > maxLat) maxLat = lat;
+      if (lat < minLat) minLat = lat;
+    }
+
+    if (maxLon - minLon > 180.0) {
+      const tmp = minLon;
+      minLon = maxLon;
+      maxLon = tmp;
+    }
+
+    return [minLat, minLon, maxLat, maxLon];
+  }
+
   /**
    * Main task entrypoint
    * @return A payload suitable for syncing via http url sync
    */
   async run() {
     const config = this.config;
     const payload = this.message.payload;
+    const meta = this.message.meta;
+    let polygons = meta.polygons;
+    if (polygons) {
+      polygons = JSON.parse(polygons);
+    }
+
+    // If a bounding box was specified use that instead of computing an mbr
+    let box = meta.box !== '{granule.box}' ? meta.box : null;
 
+    // create a list of the files to download from S3 based on the payload contents
     let files = [];
     if (Array.isArray(payload)) {
       files = files.concat(payload);
     }
     else if (payload) {
       files.push(payload);
     }
+    // Add any additional files to the file list to be downloaded - these are config files
+    // and what not that are the same for every execution of run-gdal. These are kept
+    // in S3 so they can be downloaded when this task is run.
     if (config.additional_files) {
       files = files.concat(config.additional_files);
     }
+    // We are going to copy things from S3 to be processed on our local lambda file system.
+    // The local files will be named with the names given in our configuration (this allows
+    // the gdal command arguments to be statically defined in the config and still work, e.g.,
+    // the input file will always be named 'input'). The following line is a sanity check to make
+    // sure we have defined a local file name for everything we will download from S3.
     if (files.length > config.input_filenames.length) {
       throw new Error('input_filenames do not provide enough values for input files');
     }
+    // Download everything from S3 and give them the configured file names on the local file system.
     const downloads = files.map((s3file, i) =>
       aws.downloadS3File(s3file, path.join('/tmp', config.input_filenames[i]))
     );
@@ -40,13 +97,59 @@ module.exports = class RunGdalTask extends Task {
       downloads.concat(
         this.promiseSpawn('mkdir', ['-p', 'in', 'out', 'work', 'logs'])));
 
-    for (const command of config.commands) {
-      await this.runGdalCommand(command.gdal_command, command.args);
+    // XXX Ideally this code should just execute the gdal commands that have been configured,
+    // but images crossing the anti-meridian (see GITC-567) need to be split before being
+    // processed and then merged together. This specific check makes this implementation less
+    // generic than it had been previously.
+
+    let [latBL, lonBL, latUR, lonUR] = box || RunGdalTask.computeMbr(polygons[0][0]);
+    if (lonBL > 0 && lonUR < 0) {
+      // crosses the anti-meridian
+      log.info("Splitting granule at the anti-meridian");
+      const leftLon = 179.999;
+      const rightLon = 180.001;
+
+      // change -180 to 0 to 180 to 360
+      lonUR = 360.0 + lonUR;
+
+      const leftCoords = [[lonBL, latBL], [leftLon, latBL], [leftLon, latUR], [lonBL, latUR], [lonBL, latBL]];
+      const rightCoords = [[rightLon, latBL], [lonUR, latBL], [lonUR, latUR], [rightLon, latUR], [rightLon, latBL]];
+
+      const leftMap = { type: 'Polygon', coordinates: [leftCoords] };
+      const rightMap = { type: 'Polygon', coordinates: [rightCoords] };
+
+      fs.writeFileSync('/tmp/left_side_cmr.json', JSON.stringify(leftMap));
+      fs.writeFileSync('/tmp/right_side_cmr.json', JSON.stringify(rightMap));
+
+      // DEBUG
+      let contents = fs.readFileSync('/tmp/left_side_cmr.json').toString();
+      log.info("LEFT GeoJSON FILE CONTENTS:");
+      log.info(contents);
+
+      contents = fs.readFileSync('/tmp/right_side_cmr.json').toString();
+      log.info("RIGHT GeoJSON FILE CONTENTS:");
+      log.info(contents);
+
+      for (const command of config.alternate_commands) {
+        await this.runGdalCommand(command.gdal_command, command.args);
+      }
+    }
+    else {
+      for (const command of config.commands) {
+        await this.runGdalCommand(command.gdal_command, command.args);
+      }
     }
 
     const outputPromises = config.outputs.map((output) => this.compressAndUploadOutput(output));
 
     const result = await Promise.all(outputPromises);
+
+    // clean up the directory where images are stored to prevent conlicts between lambda
+    // invocations
+    log.info('Removing /tmp/in directory');
+    rimraf('/tmp/in', () => log.info('Removed /tmp/in'))
+
+
     return result.map((obj) => ({ Key: obj[0].key, Bucket: obj[0].bucket }));
   }
 

cumulus/tasks/run-gdal/package.json

@@ -13,11 +13,21 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "scripts": {
-    "test": "echo 'no tests'",
+    "test": "ava test/*.js",
     "build": "webpack && unzip -o -q deps/lambda-gdal.zip -d dist",
     "watch": "webpack --progress -w",
     "clean": "rm -rf dist"
   },
+  "directories": {
+    "test": "test"
+  },
+  "ava": {
+    "babel": "inherit",
+    "require": [
+      "babel-polyfill",
+      "babel-register"
+    ]
+  },
   "devDependencies": {
     "babel-core": "^6.25.0",
     "babel-loader": "^6.2.4",
@@ -28,6 +38,7 @@
     "webpack": "^1.12.13"
   },
   "dependencies": {
-    "@cumulus/common": "^1.0.0"
+    "@cumulus/common": "^1.0.0",
+    "rimraf": "^2.6.2"
   }
 }

cumulus/tasks/run-gdal/test/handle-antimeridan-crossings-spec.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const test = require('ava');
+const RunGdalTask = require('../index');
+
+test('mbr - crossing prime meridian', (t) => {
+  const polyString = '-16.8145921 -17.582719 -16.803388 17.642083 -16.1463177 17.6533954 -16.15706 -17.57401 -16.8145921 -17.582719';
+  const result = RunGdalTask.computeMbr(polyString);
+  const expected = [-16.8145921, -17.582719, -16.1463177, 17.6533954];
+  t.deepEqual(result, expected);
+});
+
+test('mbr - crossing anti-meridian', (t) => {
+  const polyString = '-16.8145921 179.582719 -16.803388 -179.642083 -16.1463177 -179.6533954 -16.15706 179.57401 -16.8145921 179.582719';
+  const result = RunGdalTask.computeMbr(polyString);
+  const expected = [-16.8145921, 179.582719, -16.1463177, -179.6533954];
+  t.deepEqual(result, expected);
+});
+
+test('mbr - neither crossing prime meridian nor anti-meridian', (t) => {
+  const polyString = '-16.8145921 -17.582719 -16.803388 -7.642083 -16.1463177 -7.6533954 -16.15706 -17.57401 -16.8145921 -17.582719';
+  const result = RunGdalTask.computeMbr(polyString);
+  const expected = [-16.8145921, -17.582719, -16.1463177, -7.642083];
+  t.deepEqual(result, expected);
+});

packages/common/field-pattern.js

@@ -79,7 +79,12 @@ module.exports = class FieldPattern {
         nestedValue = nestedValue[keypath.shift()];
       }
       if (nestedValue) {
-        result = result.replace(`{${field}}`, nestedValue);
+        if (Array.isArray(nestedValue)) {
+          result = result.replace(`{${field}}`, JSON.stringify(nestedValue));
+        }
+        else {
+          result = result.replace(`{${field}}`, nestedValue);
+        }
       }
     }
     return this.stringToValue(result);

packages/common/test/config/test-collections.yml

@@ -119,6 +119,8 @@ _templates:
           event: wms-map-found
           granule_meta:
             key: '{meta.collection}/{granule.id}'
+            polygons: '{granule.polygons}'
+            box: '{granule.box}'
             granuleId: '{granule.id}'
             version: '{granule.updated}'
             date: