diff --git a/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml b/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml index ff397060470..5b8725c3460 100644 --- a/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml +++ b/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml @@ -50,6 +50,8 @@ _templates: #temporal: 2015-01-01T00:00:00Z,2015-01-02T18:39:00Z # Exactly 949 (+1 that has no day_night_flag) granule_meta: key: '{meta.collection}/{granule.id}' + polygons: '{granule.polygons}' + box: '{granule.box}' granuleId: '{granule.id}' version: '{granule.updated}' date: @@ -99,6 +101,36 @@ _templates: - Blank_RGBA_512.png - mrfgen_configuration_file.xml + alternate_commands: + - gdal_command: gdalwarp + args: [ + -of, GTiff, + -cutline, /tmp/left_side_cmr.json, + -crop_to_cutline, + -t_srs, EPSG:4326, + -tr, 0.0001373291015625, 0.0001373291015625, + -srcnodata, "0 0 0", + -dstalpha, + -r, near, + -overwrite, + input.tif, + in/processed_left_4326.tif + ] + - gdal_command: gdalwarp + args: [ + -of, GTiff, + -cutline, /tmp/right_side_cmr.json, + -crop_to_cutline, + -t_srs, EPSG:4326, + -tr, 0.0001373291015625, 0.0001373291015625, + -srcnodata, "0 0 0", + -dstalpha, + -r, near, + -overwrite, + input.tif, + in/processed_right_4326.tif + ] + commands: - gdal_command: gdalwarp args: [ diff --git a/cumulus/tasks/discover-cmr-granules/index.js b/cumulus/tasks/discover-cmr-granules/index.js index 6fe98035bd5..8047b98b7bf 100644 --- a/cumulus/tasks/discover-cmr-granules/index.js +++ b/cumulus/tasks/discover-cmr-granules/index.js @@ -28,7 +28,7 @@ function validateParameter(config, param) { * @returns {bool} - true or exception */ function validateParameters(config) { - const params = ['root', 'event', 'granule_meta', 'query']; + const params = ['root', 'granule_meta', 'query']; for (const p of params) validateParameter(config, p); return true; @@ -62,16 +62,16 @@ module.exports = class DiscoverCmrGranulesTask extends Task { const filtered = this.excludeFiltered(messages, this.config.filtered_granule_keys); // Write the messages to a DynamoDB table so we can track ingest failures - const messagePromises = filtered.map(msg => { + const messagePromises = filtered.map((msg) => { const { granuleId, version, collection } = msg.meta; const params = { TableName: this.config.ingest_tracking_table, Item: { 'granule-id': granuleId, - 'version': version, - 'collection': collection, + version: version, + collection: collection, 'ingest-start-datetime': moment().format(), - 'message': JSON.stringify(msg) + message: JSON.stringify(msg) } }; return docClient.put(params).promise(); @@ -116,6 +116,7 @@ module.exports = class DiscoverCmrGranulesTask extends Task { * * @param {string} root - The CMR root url (protocol and domain without path) * @param {Object} query - The query parameters to serialize and send to a CMR granules search + * @param {string} scrollID - The scroll id to pass to the CMR harvesting API * @returns {Array} An array of all granules matching the given query */ async cmrGranules(root, query, scrollID) { @@ -195,8 +196,9 @@ module.exports = class DiscoverCmrGranulesTask extends Task { // To use with Visual Studio Code Debugger, uncomment next block // -//global.__isDebug = true; -//const local = require('@cumulus/common/local-helpers'); -//const localTaskName = 'DiscoverCmrGranules'; -//local.setupLocalRun(module.exports.handler, local.collectionMessageInput( -// 'MOPITT_DCOSMR_LL_D_STD', localTaskName)); +// global.__isDebug = true; +// const local = require('@cumulus/common/local-helpers'); +// const localTaskName = 'DiscoverCmrGranules'; +// local.setupLocalRun(module.exports.handler, local.collectionMessageInput( +// 'AST_L1T_DAY', localTaskName, (o) => o, `${local.fileRoot()}/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml`)); + diff --git a/cumulus/tasks/discover-cmr-granules/test/discover-cmr-granules-spec.js b/cumulus/tasks/discover-cmr-granules/test/discover-cmr-granules-spec.js index df5be7b3df1..0ab822fa4d9 100644 --- a/cumulus/tasks/discover-cmr-granules/test/discover-cmr-granules-spec.js +++ b/cumulus/tasks/discover-cmr-granules/test/discover-cmr-granules-spec.js @@ -35,15 +35,6 @@ test('check with invalid root parameter', async (t) => { t.is(errors, 'Undefined root parameter'); }); -test('check with invalid event parameter', async (t) => { - // Remove event parameter for DiscoverCmrGranules Task - const newPayload = _.cloneDeep(message); //Object assign will not work here - delete newPayload.workflow_config_template.DiscoverCmrGranules.event; - - const [errors] = await testHelpers.run(DiscoverCmrGranules, newPayload); - t.is(errors, 'Undefined event parameter'); -}); - test('check with invalid granule_meta parameter', async (t) => { // Remove granule_meta parameter for DiscoverCmrGranules Task const newPayload = _.cloneDeep(message); //Object assign will not work here diff --git a/cumulus/tasks/run-gdal/index.js b/cumulus/tasks/run-gdal/index.js index 851dcd8ff1e..52f7a34c2f9 100644 --- a/cumulus/tasks/run-gdal/index.js +++ b/cumulus/tasks/run-gdal/index.js @@ -6,11 +6,50 @@ const log = require('@cumulus/common/log'); const fs = require('fs'); const path = require('path'); const spawn = require('child_process').spawn; +const rimraf = require('rimraf'); /** * */ module.exports = class RunGdalTask extends Task { + + /** + * Compute the minimum bounding rectangle (MBR) for the given polygon. Assumes the maximum + * longitudinal distance * between points is less than 180 degrees. + * NOTE: This will NOT work for polygons covering poles. + * @param {string} polyString A polygon from a CMR metadata response. This has the form + * "lat_0 lon_0 lat_1 lon_1 ... lat_n lon_n lat_0 lon_0" + * @returns {Array} An array containing the mbr in the form [latBL, lonBL, latUR, lonRU] where + * latBL = latitude of the bottom left corner + * lonBL = longitude of the bottom left corner + * latUR = latitude of the upper right corner + * lonUR = longitude of the upper right corner + */ + static computeMbr(polyString) { + const coords = polyString.split(' '); + let minLat = 360.0; + let minLon = 360.0; + let maxLat = -360.0; + let maxLon = -360.0; + for (let i = 1; i < coords.length / 2; i++) { + const lat = parseFloat(coords[i * 2]); + const lon = parseFloat(coords[(i * 2) + 1]); + + if (lon > maxLon) maxLon = lon; + if (lon < minLon) minLon = lon; + if (lat > maxLat) maxLat = lat; + if (lat < minLat) minLat = lat; + } + + if (maxLon - minLon > 180.0) { + const tmp = minLon; + minLon = maxLon; + maxLon = tmp; + } + + return [minLat, minLon, maxLat, maxLon]; + } + /** * Main task entrypoint * @return A payload suitable for syncing via http url sync @@ -18,7 +57,16 @@ module.exports = class RunGdalTask extends Task { async run() { const config = this.config; const payload = this.message.payload; + const meta = this.message.meta; + let polygons = meta.polygons; + if (polygons) { + polygons = JSON.parse(polygons); + } + + // If a bounding box was specified use that instead of computing an mbr + let box = meta.box !== '{granule.box}' ? meta.box : null; + // create a list of the files to download from S3 based on the payload contents let files = []; if (Array.isArray(payload)) { files = files.concat(payload); @@ -26,12 +74,21 @@ module.exports = class RunGdalTask extends Task { else if (payload) { files.push(payload); } + // Add any additional files to the file list to be downloaded - these are config files + // and what not that are the same for every execution of run-gdal. These are kept + // in S3 so they can be downloaded when this task is run. if (config.additional_files) { files = files.concat(config.additional_files); } + // We are going to copy things from S3 to be processed on our local lambda file system. + // The local files will be named with the names given in our configuration (this allows + // the gdal command arguments to be statically defined in the config and still work, e.g., + // the input file will always be named 'input'). The following line is a sanity check to make + // sure we have defined a local file name for everything we will download from S3. if (files.length > config.input_filenames.length) { throw new Error('input_filenames do not provide enough values for input files'); } + // Download everything from S3 and give them the configured file names on the local file system. const downloads = files.map((s3file, i) => aws.downloadS3File(s3file, path.join('/tmp', config.input_filenames[i])) ); @@ -40,13 +97,59 @@ module.exports = class RunGdalTask extends Task { downloads.concat( this.promiseSpawn('mkdir', ['-p', 'in', 'out', 'work', 'logs']))); - for (const command of config.commands) { - await this.runGdalCommand(command.gdal_command, command.args); + // XXX Ideally this code should just execute the gdal commands that have been configured, + // but images crossing the anti-meridian (see GITC-567) need to be split before being + // processed and then merged together. This specific check makes this implementation less + // generic than it had been previously. + + let [latBL, lonBL, latUR, lonUR] = box || RunGdalTask.computeMbr(polygons[0][0]); + if (lonBL > 0 && lonUR < 0) { + // crosses the anti-meridian + log.info("Splitting granule at the anti-meridian"); + const leftLon = 179.999; + const rightLon = 180.001; + + // change -180 to 0 to 180 to 360 + lonUR = 360.0 + lonUR; + + const leftCoords = [[lonBL, latBL], [leftLon, latBL], [leftLon, latUR], [lonBL, latUR], [lonBL, latBL]]; + const rightCoords = [[rightLon, latBL], [lonUR, latBL], [lonUR, latUR], [rightLon, latUR], [rightLon, latBL]]; + + const leftMap = { type: 'Polygon', coordinates: [leftCoords] }; + const rightMap = { type: 'Polygon', coordinates: [rightCoords] }; + + fs.writeFileSync('/tmp/left_side_cmr.json', JSON.stringify(leftMap)); + fs.writeFileSync('/tmp/right_side_cmr.json', JSON.stringify(rightMap)); + + // DEBUG + let contents = fs.readFileSync('/tmp/left_side_cmr.json').toString(); + log.info("LEFT GeoJSON FILE CONTENTS:"); + log.info(contents); + + contents = fs.readFileSync('/tmp/right_side_cmr.json').toString(); + log.info("RIGHT GeoJSON FILE CONTENTS:"); + log.info(contents); + + for (const command of config.alternate_commands) { + await this.runGdalCommand(command.gdal_command, command.args); + } + } + else { + for (const command of config.commands) { + await this.runGdalCommand(command.gdal_command, command.args); + } } const outputPromises = config.outputs.map((output) => this.compressAndUploadOutput(output)); const result = await Promise.all(outputPromises); + + // clean up the directory where images are stored to prevent conlicts between lambda + // invocations + log.info('Removing /tmp/in directory'); + rimraf('/tmp/in', () => log.info('Removed /tmp/in')) + + return result.map((obj) => ({ Key: obj[0].key, Bucket: obj[0].bucket })); } diff --git a/cumulus/tasks/run-gdal/package.json b/cumulus/tasks/run-gdal/package.json index 926899dc8df..80f4a999999 100644 --- a/cumulus/tasks/run-gdal/package.json +++ b/cumulus/tasks/run-gdal/package.json @@ -13,11 +13,21 @@ "author": "Cumulus Authors", "license": "Apache-2.0", "scripts": { - "test": "echo 'no tests'", + "test": "ava test/*.js", "build": "webpack && unzip -o -q deps/lambda-gdal.zip -d dist", "watch": "webpack --progress -w", "clean": "rm -rf dist" }, + "directories": { + "test": "test" + }, + "ava": { + "babel": "inherit", + "require": [ + "babel-polyfill", + "babel-register" + ] + }, "devDependencies": { "babel-core": "^6.25.0", "babel-loader": "^6.2.4", @@ -28,6 +38,7 @@ "webpack": "^1.12.13" }, "dependencies": { - "@cumulus/common": "^1.0.0" + "@cumulus/common": "^1.0.0", + "rimraf": "^2.6.2" } } diff --git a/cumulus/tasks/run-gdal/test/handle-antimeridan-crossings-spec.js b/cumulus/tasks/run-gdal/test/handle-antimeridan-crossings-spec.js new file mode 100644 index 00000000000..ed7970515c8 --- /dev/null +++ b/cumulus/tasks/run-gdal/test/handle-antimeridan-crossings-spec.js @@ -0,0 +1,25 @@ +'use strict'; + +const test = require('ava'); +const RunGdalTask = require('../index'); + +test('mbr - crossing prime meridian', (t) => { + const polyString = '-16.8145921 -17.582719 -16.803388 17.642083 -16.1463177 17.6533954 -16.15706 -17.57401 -16.8145921 -17.582719'; + const result = RunGdalTask.computeMbr(polyString); + const expected = [-16.8145921, -17.582719, -16.1463177, 17.6533954]; + t.deepEqual(result, expected); +}); + +test('mbr - crossing anti-meridian', (t) => { + const polyString = '-16.8145921 179.582719 -16.803388 -179.642083 -16.1463177 -179.6533954 -16.15706 179.57401 -16.8145921 179.582719'; + const result = RunGdalTask.computeMbr(polyString); + const expected = [-16.8145921, 179.582719, -16.1463177, -179.6533954]; + t.deepEqual(result, expected); +}); + +test('mbr - neither crossing prime meridian nor anti-meridian', (t) => { + const polyString = '-16.8145921 -17.582719 -16.803388 -7.642083 -16.1463177 -7.6533954 -16.15706 -17.57401 -16.8145921 -17.582719'; + const result = RunGdalTask.computeMbr(polyString); + const expected = [-16.8145921, -17.582719, -16.1463177, -7.642083]; + t.deepEqual(result, expected); +}); diff --git a/packages/common/field-pattern.js b/packages/common/field-pattern.js index c9fe47b5cbb..644afbdfb05 100644 --- a/packages/common/field-pattern.js +++ b/packages/common/field-pattern.js @@ -79,7 +79,12 @@ module.exports = class FieldPattern { nestedValue = nestedValue[keypath.shift()]; } if (nestedValue) { - result = result.replace(`{${field}}`, nestedValue); + if (Array.isArray(nestedValue)) { + result = result.replace(`{${field}}`, JSON.stringify(nestedValue)); + } + else { + result = result.replace(`{${field}}`, nestedValue); + } } } return this.stringToValue(result); diff --git a/packages/common/test/config/test-collections.yml b/packages/common/test/config/test-collections.yml index 849b6df5970..cfeb60a8440 100644 --- a/packages/common/test/config/test-collections.yml +++ b/packages/common/test/config/test-collections.yml @@ -119,6 +119,8 @@ _templates: event: wms-map-found granule_meta: key: '{meta.collection}/{granule.id}' + polygons: '{granule.polygons}' + box: '{granule.box}' granuleId: '{granule.id}' version: '{granule.updated}' date: