GITC-602: Add changes to S3 permissions to support OnEarth reads (#240)

* GITC-602: Add changes to S3 permissions to support OnEarth reads * GITC-602: Remove VIIRS entries in test-collections.yml * GITC-602: Fix failing test caused by bad config
nasa · Mar 13, 2018 · b22342d · b22342d
1 parent 5fb5888
commit b22342d
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 141 deletions.
diff --git a/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml b/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml
@@ -95,11 +95,20 @@ _templates:
               Key: "ingest/empty_tiles/Blank_RGBA_512.png"
             - Bucket: "{resources.buckets.config}"
               Key: "ingest/mrfconfig/ast_l1t.xml"
+            - Bucket: "{resources.buckets.config}"
+              Key: "ingest/mrfconfig/ast_l1t_1.xml"
+            - Bucket: "{resources.buckets.config}"
+              Key: "ingest/mrfconfig/ast_l1t_2.xml"
+            - Bucket: "{resources.buckets.config}"
+              Key: "ingest/mrfconfig/ast_l1t_3.xml"
 
           input_filenames:
             - input.tif
             - Blank_RGBA_512.png
             - mrfgen_configuration_file.xml
+            - mrfgen_configuration_file_1.xml
+            - mrfgen_configuration_file_2.xml
+            - mrfgen_configuration_file_3.xml
 
           alternate_commands:
             - gdal_command: gdalwarp
@@ -198,16 +207,16 @@ _templates:
             - filename: out/out.mrf
               dest:
                 Bucket: '{resources.buckets.public}'
-                Key: 'archive/geographic/aster_l1t/mrfs/AST_L1T_LL/AST_L1T_LL{meta.date.dateTimeStamp}_.mrf'
+                Key: 'epsg{meta.epsg}/{meta.collection}/{meta.date.year}/#hash-{meta.collection}-{meta.date.dateTimeStamp}.mrf'
             - filename: out/out.idx
               compress: true
               dest:
                 Bucket: '{resources.buckets.public}'
-                Key: 'archive/geographic/aster_l1t/mrfs/AST_L1T_LL/AST_L1T_LL{meta.date.dateTimeStamp}_.idx.tgz'
+                Key: 'epsg{meta.epsg}/{meta.collection}/{meta.date.year}/#hash-{meta.collection}-{meta.date.dateTimeStamp}.idx.tgz'
             - filename: out/out.ppg
               dest:
                 Bucket: '{resources.buckets.public}'
-                Key: 'archive/geographic/aster_l1t/mrfs/AST_L1T_LL/AST_L1T_LL{meta.date.dateTimeStamp}_.ppg'
+                Key: 'epsg{meta.epsg}/{meta.collection}/{meta.date.year}/#hash-{meta.collection}-{meta.date.dateTimeStamp}.ppg'
 
 _workflow_meta:
   - id: DiscoverASTER
@@ -299,10 +308,12 @@ collections:
       collection: AST_L1T_LL
       day_night_flag: 'day'
       url_pattern: '_V.tif$'
+      epsg: 4326
 
   - <<: *AST_L1T
     id: AST_L1T_NIGHT
     meta:
       collection: AST_L1T_LL
       day_night_flag: 'night'
-      url_pattern: '_T.tif$'
+      url_pattern: '_T.tif$'
+      epsg: 4326
diff --git a/cumulus/tasks/run-gdal/index.js b/cumulus/tasks/run-gdal/index.js
@@ -7,6 +7,8 @@ const fs = require('fs');
 const path = require('path');
 const spawn = require('child_process').spawn;
 const rimraf = require('rimraf');
+const md5 = require('md5');
+const currentDayNumber = require('current-day-number');
 
 /**
  *
@@ -164,11 +166,23 @@ module.exports = class RunGdalTask extends Task {
       await this.promiseSpawn('tar', ['cvpzfS', filename, output.filename]);
     }
 
+
+    const outputKey = output.dest.Key;
+    const regex = /(epsg.*?\/.*?\/\d{4})\/#hash-(.*?)-(\d{4})(\d{2})(\d{2})(\d{6})\.(.*?)$/;
+    const [_, base, collection, year, month, day, time, ext] = regex.exec(outputKey);
+
+    const dayOfYear = currentDayNumber(`${month}/${day}/${year}`).toString().padStart(3, '0');
+    const dateTime = `${year}${dayOfYear}${time}`;
+
+    const hash = md5(`${collection}-${dateTime}`).slice(0, 4);
+
+    const s3Key = `${base}/${hash}-${collection}-${dateTime}.${ext}`;
+
     return aws.uploadS3Files([{
       filename: path.join('/tmp', filename),
       bucket: output.dest.Bucket,
-      key: output.dest.Key
-    }]);
+      key: s3Key
+    }], null, null, { ACL: 'public-read' });
   }
 
   /**
@@ -228,3 +242,10 @@ module.exports = class RunGdalTask extends Task {
     return RunGdalTask.handle(...args);
   }
 };
+
+// global.__isDebug = true;
+// const local = require('@cumulus/common/local-helpers');
+// const localTaskName = 'GenerateMrfFromTiff';
+// local.setupLocalRun(module.exports.handler, local.collectionMessageInput(
+//  'AST_L1T_DAY', localTaskName, (o) => o, `${local.fileRoot()}/cumulus/tasks/copy-idx-from-s3-to-efs/test/ast_l1t.yml`));
+
diff --git a/cumulus/tasks/run-gdal/package.json b/cumulus/tasks/run-gdal/package.json
@@ -39,6 +39,7 @@
   },
   "dependencies": {
     "@cumulus/common": "^1.0.0",
-    "rimraf": "^2.6.2"
+    "rimraf": "^2.6.2",
+    "current-day-number": "^1.0.6"
   }
 }
diff --git a/packages/common/test/config/test-collections.yml b/packages/common/test/config/test-collections.yml
@@ -1,8 +1,7 @@
 providers:
-  - id: MODAPS
-    config: {}
   - id: LARC
     config: {}
+
   - id: SIPSPROV
     config:
       gateway_config:
@@ -16,60 +15,6 @@ providers:
         num_connections: 2
 _templates:
   _collection_types:
-    - &VIIRS
-      provider_id: MODAPS
-
-      workflow: !GitcResource DiscoverVIIRS
-
-      trigger:
-        type: interval
-        period_s: 900
-
-      workflow_config_template:
-        DiscoverHttpTiles:
-          type: apache-index
-          connections: 10
-          root: 'https://lance3.modaps.eosdis.nasa.gov/imagery/elements/VIIRS/{meta.collection}/'
-          match: 'https://lance3.modaps.eosdis.nasa.gov/imagery/elements/VIIRS/{meta.collection}/{_}/{jday}/{file}.{ext}'
-          required:
-            -
-              group: file
-              prop: ext
-              values:
-                - jgw
-                - jpg
-                - txt
-          group_meta:
-            key: 'VIIRS/{meta.collection}/{match.jday}'
-            date:
-              type: date
-              value: '{match.jday}'
-          group_by: 'VIIRS/{meta.collection}/{match.jday}'
-
-        TriggerIngest:
-          workflow: !GitcResource IngestVIIRS
-
-        SyncHttpUrls:
-          connections: 5
-          output:
-            bucket: '{resources.buckets.private}'
-            key_prefix: 'sources/EPSG{meta.epsg}/{meta.key}'
-
-        MRFGen:
-          date: '{meta.date.year}{meta.date.month}{meta.date.day}'
-          epsg: '{meta.epsg}'
-          zoom: '{meta.zoom}'
-          mrfgen:
-            mrf_compression_type: JPEG
-            source_epsg: '{meta.epsg}'
-            mrf_merge: false
-            mrf_nocopy: true
-            overview_resampling: average
-            resize_resampling: average
-            parameter_name: '{meta.parameter_name}'
-          output:
-            bucket: '{resources.buckets.public}'
-            key_prefix: 'EPSG{meta.epsg}/{meta.collection}/{meta.date.year}'
 
     - &SIPSTEST
       provider_id: SIPSPROV
@@ -168,15 +113,6 @@ _templates:
             Key: 'EPSG{meta.epsg}/{meta.collection}/{meta.date.year}'
 
 _workflow_meta:
-  - id: DiscoverVIIRS
-    name: VIIRS Discovery
-    arn: !GitcResource DiscoverVIIRS
-    type: discovery
-  - id: IngestVIIRS
-    name: VIIRS Ingest
-    arn: !GitcResource IngestVIIRS
-    type: ingest
-    parent: DiscoverVIIRS
   - id: DiscoverMOPITT
     name: MOPITT Discovery
     arn: !GitcResource DiscoverMOPITT
@@ -197,52 +133,6 @@ _workflow_meta:
     parent: DiscoverPdrsSIPSTEST
 
 workflows:
-  DiscoverVIIRS:
-    Comment: VIIRS Discovery
-    StartAt: DiscoverHttpTiles
-    States:
-      DiscoverHttpTiles:
-        Type: Task
-        Resource: !GitcResource DiscoverHttpTilesFn.Arn
-        Next: TriggerIngest
-
-      TriggerIngest:
-        Type: Task
-        Resource: !GitcResource TriggerIngestFn.Arn
-        End: true
-
-  IngestVIIRS:
-    Comment: VIIRS Ingest
-    StartAt: SyncHttpUrls
-    States:
-      SyncHttpUrls:
-        Type: Task
-        Resource: !GitcResource SyncHttpUrlsFn.Arn
-        Next: ChooseSyncContinuation
-      ChooseSyncContinuation:
-        Type: Choice
-        Choices:
-          - Variable: $.exception
-            StringEquals: NotNeededWorkflowError
-            Next: NoUpdates
-          - Variable: $.exception
-            StringEquals: IncompleteWorkflowError
-            Next: SyncHttpUrls
-          - Variable: $.exception
-            StringEquals: None
-            Next: MRFGen
-        Default: UnhandledSyncError
-      UnhandledSyncError:
-        Type: Fail
-        Cause: Unhandled Sync Error
-      MRFGen:
-        Type: Task
-        Resource: !GitcResource GenerateMrfActivity
-        End: true
-      NoUpdates:
-        Type: Pass
-        Result: No Updates Necessary
-        End: true
 
   DiscoverMOPITT:
     Comment: MOPITT Discovery
@@ -368,29 +258,6 @@ workflows:
 
 
 collections:
-  - <<: *VIIRS
-    id: VNGCR_LQD_C1
-    meta:
-      collection: VNGCR_LQD_C1
-      parameter_name: VNGCR_LQD_I1-M4-M3_NRT
-      epsg: 4326
-      zoom: 9
-
-  - <<: *VIIRS
-    id: VNGCR_SQD_C1
-    meta:
-      collection: VNGCR_SQD_C1
-      parameter_name: VNGCR_SQD_I1-M4-M3_NRT
-      epsg: 3031
-      zoom: 6
-
-  - <<: *VIIRS
-    id: VNGCR_NQD_C1
-    meta:
-      collection: VNGCR_NQD_C1
-      parameter_name: VNGCR_NQD_I1-M4-M3_NRT
-      epsg: 3413
-      zoom: 6
 
   - <<: *MOPITT
     id: MOPITT_DCOSMR_LL_D_STD