From e73794f6a53b3f56d01161eb5d41b694bbeb50fc Mon Sep 17 00:00:00 2001 From: James Norton Date: Mon, 12 Mar 2018 14:42:26 -0400 Subject: [PATCH 1/3] GITC-602: Add changes to S3 permissions to support OnEarth reads --- .../copy-idx-from-s3-to-efs/test/ast_l1t.yml | 19 +++++-- cumulus/tasks/run-gdal/index.js | 25 +++++++- cumulus/tasks/run-gdal/package.json | 3 +- .../common/test/config/test-collections.yml | 57 +------------------ 4 files changed, 41 insertions(+), 63 deletions(-) diff --git a/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml b/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml index 5b8725c3460..4e07cf26243 100644 --- a/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml +++ b/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml @@ -95,11 +95,20 @@ _templates: Key: "ingest/empty_tiles/Blank_RGBA_512.png" - Bucket: "{resources.buckets.config}" Key: "ingest/mrfconfig/ast_l1t.xml" + - Bucket: "{resources.buckets.config}" + Key: "ingest/mrfconfig/ast_l1t_1.xml" + - Bucket: "{resources.buckets.config}" + Key: "ingest/mrfconfig/ast_l1t_2.xml" + - Bucket: "{resources.buckets.config}" + Key: "ingest/mrfconfig/ast_l1t_3.xml" input_filenames: - input.tif - Blank_RGBA_512.png - mrfgen_configuration_file.xml + - mrfgen_configuration_file_1.xml + - mrfgen_configuration_file_2.xml + - mrfgen_configuration_file_3.xml alternate_commands: - gdal_command: gdalwarp @@ -198,16 +207,16 @@ _templates: - filename: out/out.mrf dest: Bucket: '{resources.buckets.public}' - Key: 'archive/geographic/aster_l1t/mrfs/AST_L1T_LL/AST_L1T_LL{meta.date.dateTimeStamp}_.mrf' + Key: 'epsg{meta.epsg}/{meta.collection}/{meta.date.year}/#hash-{meta.collection}-{meta.date.dateTimeStamp}.mrf' - filename: out/out.idx compress: true dest: Bucket: '{resources.buckets.public}' - Key: 'archive/geographic/aster_l1t/mrfs/AST_L1T_LL/AST_L1T_LL{meta.date.dateTimeStamp}_.idx.tgz' + Key: 'epsg{meta.epsg}/{meta.collection}/{meta.date.year}/#hash-{meta.collection}-{meta.date.dateTimeStamp}.idx.tgz' - filename: out/out.ppg dest: Bucket: '{resources.buckets.public}' - Key: 'archive/geographic/aster_l1t/mrfs/AST_L1T_LL/AST_L1T_LL{meta.date.dateTimeStamp}_.ppg' + Key: 'epsg{meta.epsg}/{meta.collection}/{meta.date.year}/#hash-{meta.collection}-{meta.date.dateTimeStamp}.ppg' _workflow_meta: - id: DiscoverASTER @@ -299,10 +308,12 @@ collections: collection: AST_L1T_LL day_night_flag: 'day' url_pattern: '_V.tif$' + epsg: 4326 - <<: *AST_L1T id: AST_L1T_NIGHT meta: collection: AST_L1T_LL day_night_flag: 'night' - url_pattern: '_T.tif$' \ No newline at end of file + url_pattern: '_T.tif$' + epsg: 4326 \ No newline at end of file diff --git a/cumulus/tasks/run-gdal/index.js b/cumulus/tasks/run-gdal/index.js index 52f7a34c2f9..a0d1a081889 100644 --- a/cumulus/tasks/run-gdal/index.js +++ b/cumulus/tasks/run-gdal/index.js @@ -7,6 +7,8 @@ const fs = require('fs'); const path = require('path'); const spawn = require('child_process').spawn; const rimraf = require('rimraf'); +const md5 = require('md5'); +const currentDayNumber = require('current-day-number'); /** * @@ -164,11 +166,23 @@ module.exports = class RunGdalTask extends Task { await this.promiseSpawn('tar', ['cvpzfS', filename, output.filename]); } + + const outputKey = output.dest.Key; + const regex = /(epsg.*?\/.*?\/\d{4})\/#hash-(.*?)-(\d{4})(\d{2})(\d{2})(\d{6})\.(.*?)$/; + const [_, base, collection, year, month, day, time, ext] = regex.exec(outputKey); + + const dayOfYear = currentDayNumber(`${month}/${day}/${year}`).toString().padStart(3, '0'); + const dateTime = `${year}${dayOfYear}${time}`; + + const hash = md5(`${collection}-${dateTime}`).slice(0, 4); + + const s3Key = `${base}/${hash}-${collection}-${dateTime}.${ext}`; + return aws.uploadS3Files([{ filename: path.join('/tmp', filename), bucket: output.dest.Bucket, - key: output.dest.Key - }]); + key: s3Key + }], null, null, { ACL: 'public-read' }); } /** @@ -228,3 +242,10 @@ module.exports = class RunGdalTask extends Task { return RunGdalTask.handle(...args); } }; + +// global.__isDebug = true; +// const local = require('@cumulus/common/local-helpers'); +// const localTaskName = 'GenerateMrfFromTiff'; +// local.setupLocalRun(module.exports.handler, local.collectionMessageInput( +// 'AST_L1T_DAY', localTaskName, (o) => o, `${local.fileRoot()}/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml`)); + diff --git a/cumulus/tasks/run-gdal/package.json b/cumulus/tasks/run-gdal/package.json index 80f4a999999..23054566b15 100644 --- a/cumulus/tasks/run-gdal/package.json +++ b/cumulus/tasks/run-gdal/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@cumulus/common": "^1.0.0", - "rimraf": "^2.6.2" + "rimraf": "^2.6.2", + "current-day-number": "^1.0.6" } } diff --git a/packages/common/test/config/test-collections.yml b/packages/common/test/config/test-collections.yml index cfeb60a8440..57ee04419c7 100644 --- a/packages/common/test/config/test-collections.yml +++ b/packages/common/test/config/test-collections.yml @@ -1,8 +1,7 @@ providers: - id: MODAPS config: {} - - id: LARC - config: {} + - id: SIPSPROV config: gateway_config: @@ -16,60 +15,6 @@ providers: num_connections: 2 _templates: _collection_types: - - &VIIRS - provider_id: MODAPS - - workflow: !GitcResource DiscoverVIIRS - - trigger: - type: interval - period_s: 900 - - workflow_config_template: - DiscoverHttpTiles: - type: apache-index - connections: 10 - root: 'https://lance3.modaps.eosdis.nasa.gov/imagery/elements/VIIRS/{meta.collection}/' - match: 'https://lance3.modaps.eosdis.nasa.gov/imagery/elements/VIIRS/{meta.collection}/{_}/{jday}/{file}.{ext}' - required: - - - group: file - prop: ext - values: - - jgw - - jpg - - txt - group_meta: - key: 'VIIRS/{meta.collection}/{match.jday}' - date: - type: date - value: '{match.jday}' - group_by: 'VIIRS/{meta.collection}/{match.jday}' - - TriggerIngest: - workflow: !GitcResource IngestVIIRS - - SyncHttpUrls: - connections: 5 - output: - bucket: '{resources.buckets.private}' - key_prefix: 'sources/EPSG{meta.epsg}/{meta.key}' - - MRFGen: - date: '{meta.date.year}{meta.date.month}{meta.date.day}' - epsg: '{meta.epsg}' - zoom: '{meta.zoom}' - mrfgen: - mrf_compression_type: JPEG - source_epsg: '{meta.epsg}' - mrf_merge: false - mrf_nocopy: true - overview_resampling: average - resize_resampling: average - parameter_name: '{meta.parameter_name}' - output: - bucket: '{resources.buckets.public}' - key_prefix: 'EPSG{meta.epsg}/{meta.collection}/{meta.date.year}' - &SIPSTEST provider_id: SIPSPROV From 37bc48fb94ba32ebec48f6adfeb31c17d536d34d Mon Sep 17 00:00:00 2001 From: James Norton Date: Mon, 12 Mar 2018 15:14:16 -0400 Subject: [PATCH 2/3] GITC-602: Remove VIIRS entries in test-collections.yml --- .../common/test/config/test-collections.yml | 78 ------------------- 1 file changed, 78 deletions(-) diff --git a/packages/common/test/config/test-collections.yml b/packages/common/test/config/test-collections.yml index 57ee04419c7..666f354504a 100644 --- a/packages/common/test/config/test-collections.yml +++ b/packages/common/test/config/test-collections.yml @@ -113,15 +113,6 @@ _templates: Key: 'EPSG{meta.epsg}/{meta.collection}/{meta.date.year}' _workflow_meta: - - id: DiscoverVIIRS - name: VIIRS Discovery - arn: !GitcResource DiscoverVIIRS - type: discovery - - id: IngestVIIRS - name: VIIRS Ingest - arn: !GitcResource IngestVIIRS - type: ingest - parent: DiscoverVIIRS - id: DiscoverMOPITT name: MOPITT Discovery arn: !GitcResource DiscoverMOPITT @@ -142,52 +133,6 @@ _workflow_meta: parent: DiscoverPdrsSIPSTEST workflows: - DiscoverVIIRS: - Comment: VIIRS Discovery - StartAt: DiscoverHttpTiles - States: - DiscoverHttpTiles: - Type: Task - Resource: !GitcResource DiscoverHttpTilesFn.Arn - Next: TriggerIngest - - TriggerIngest: - Type: Task - Resource: !GitcResource TriggerIngestFn.Arn - End: true - - IngestVIIRS: - Comment: VIIRS Ingest - StartAt: SyncHttpUrls - States: - SyncHttpUrls: - Type: Task - Resource: !GitcResource SyncHttpUrlsFn.Arn - Next: ChooseSyncContinuation - ChooseSyncContinuation: - Type: Choice - Choices: - - Variable: $.exception - StringEquals: NotNeededWorkflowError - Next: NoUpdates - - Variable: $.exception - StringEquals: IncompleteWorkflowError - Next: SyncHttpUrls - - Variable: $.exception - StringEquals: None - Next: MRFGen - Default: UnhandledSyncError - UnhandledSyncError: - Type: Fail - Cause: Unhandled Sync Error - MRFGen: - Type: Task - Resource: !GitcResource GenerateMrfActivity - End: true - NoUpdates: - Type: Pass - Result: No Updates Necessary - End: true DiscoverMOPITT: Comment: MOPITT Discovery @@ -313,29 +258,6 @@ workflows: collections: - - <<: *VIIRS - id: VNGCR_LQD_C1 - meta: - collection: VNGCR_LQD_C1 - parameter_name: VNGCR_LQD_I1-M4-M3_NRT - epsg: 4326 - zoom: 9 - - - <<: *VIIRS - id: VNGCR_SQD_C1 - meta: - collection: VNGCR_SQD_C1 - parameter_name: VNGCR_SQD_I1-M4-M3_NRT - epsg: 3031 - zoom: 6 - - - <<: *VIIRS - id: VNGCR_NQD_C1 - meta: - collection: VNGCR_NQD_C1 - parameter_name: VNGCR_NQD_I1-M4-M3_NRT - epsg: 3413 - zoom: 6 - <<: *MOPITT id: MOPITT_DCOSMR_LL_D_STD From de0bc11b9dee7eaf7b580719dbeb03d4c172eb49 Mon Sep 17 00:00:00 2001 From: James Norton Date: Mon, 12 Mar 2018 18:06:04 -0400 Subject: [PATCH 3/3] GITC-602: Fix failing test caused by bad config --- packages/common/test/config/test-collections.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/common/test/config/test-collections.yml b/packages/common/test/config/test-collections.yml index 666f354504a..89a34923b4c 100644 --- a/packages/common/test/config/test-collections.yml +++ b/packages/common/test/config/test-collections.yml @@ -1,5 +1,5 @@ providers: - - id: MODAPS + - id: LARC config: {} - id: SIPSPROV