Merge pull request #3415 from nasa/release-15.0.4

Release 15.0.4
nasa · Jun 24, 2023 · f461e95 · f461e95
2 parents 5fd5feb + 9251d6d
commit f461e95
Show file tree

Hide file tree

Showing 91 changed files with 642 additions and 454 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,28 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+## [v15.0.4] 2023-06-23
+
+### Changed
+
+- **CUMULUS-3307**
+  - Pinned cumulus dependency on `pg` to `v8.10.x`
+
+### Fixed
+
+- **CUMULUS-3115**
+  - Fixed DiscoverGranules' workflow's duplicateHandling when set to `skip` or `error` to stop retrying
+    after receiving a 404 Not Found Response Error from the `cumulus-api`.
+- **CUMULUS-3315**
+  - Update CI scripts to use shell logic/GNU timeout to bound test timeouts
+    instead of NPM `parallel` package, as timeouts were not resulting in
+    integration test failure
+- **CUMULUS-3223**
+  - Update `@cumulus/cmrjs/cmr-utils.getGranuleTemporalInfo` to handle the error when the cmr file s3url is not available
+  - Update `sfEventSqsToDbRecords` lambda to return [partial batch failure](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting),
+    and only reprocess messages when cumulus message can't be retrieved from the execution events.
+  - Update `@cumulus/cumulus-message-adapter-js` to `2.0.5` for all cumulus tasks
+
 ## [v15.0.3] 2023-04-28
 
 ### Fixed
@@ -6913,7 +6935,8 @@ Note: There was an issue publishing 1.12.0. Upgrade to 1.12.1.
 
 ## [v1.0.0] - 2018-02-23
 
-[unreleased]: https://github.com/nasa/cumulus/compare/v15.0.3...HEAD
+[unreleased]: https://github.com/nasa/cumulus/compare/v15.0.4...HEAD
+[v15.0.4]: https://github.com/nasa/cumulus/compare/v15.0.3...v15.0.4
 [v15.0.3]: https://github.com/nasa/cumulus/compare/v15.0.2...v15.0.3
 [v15.0.2]: https://github.com/nasa/cumulus/compare/v15.0.1...v15.0.2
 [v15.0.1]: https://github.com/nasa/cumulus/compare/v15.0.0...v15.0.1

diff --git a/docs/deployment/README.md b/docs/deployment/README.md
@@ -523,7 +523,7 @@ Deploy your dashboard to S3 bucket from the `cumulus-dashboard` directory:
 Using AWS CLI:
 
 ```bash
-  aws s3 sync dist s3://<prefix>-dashboard --acl public-read
+  aws s3 sync dist s3://<prefix>-dashboard
 ```
 
 From the S3 Console:

diff --git a/example/config.yml b/example/config.yml
@@ -34,3 +34,6 @@ vkn-tf:
 
 np:
   bucket: npauzenga-internal
+
+nnaga-tf:
+  bucket: nnaga-internal
diff --git a/example/lambdas/asyncOperations/package.json b/example/lambdas/asyncOperations/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-async-operations",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "AsyncOperations Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/lambdas/ftpPopulateTestLambda/package.json b/example/lambdas/ftpPopulateTestLambda/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/ftp-populate-test-lambda",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "FTP Population Utility Lambda",
   "main": "index.js",
   "private": true,
@@ -19,12 +19,12 @@
     "access": "private"
   },
   "dependencies": {
-    "@cumulus/api": "15.0.3",
-    "@cumulus/api-client": "15.0.3",
-    "@cumulus/common": "15.0.3",
-    "@cumulus/integration-tests": "15.0.3",
-    "@cumulus/logger": "15.0.3",
-    "@cumulus/test-data": "15.0.3",
+    "@cumulus/api": "15.0.4",
+    "@cumulus/api-client": "15.0.4",
+    "@cumulus/common": "15.0.4",
+    "@cumulus/integration-tests": "15.0.4",
+    "@cumulus/logger": "15.0.4",
+    "@cumulus/test-data": "15.0.4",
     "aws-sdk": "^2.585.0",
     "fs-extra": "^9.0.0",
     "jsftp": "https://github.com/jkovarik/jsftp.git#add_288",

diff --git a/example/lambdas/lzardsClientTest/package.json b/example/lambdas/lzardsClientTest/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-lzards-api-lambda",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "LZARDS API Client Test Lambda",
   "private": true,
   "engines": {
@@ -20,7 +20,7 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/logger": "15.0.3",
-    "@cumulus/lzards-api-client": "15.0.3"
+    "@cumulus/logger": "15.0.4",
+    "@cumulus/lzards-api-client": "15.0.4"
   }
 }
diff --git a/example/lambdas/python-processing/package.json b/example/lambdas/python-processing/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-process-activity",
   "private": true,
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "Python reference activity",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity",
   "repository": {

diff --git a/example/lambdas/python-reference-activity/package.json b/example/lambdas/python-reference-activity/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-reference-activity",
   "private": true,
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "Python reference activity",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity",
   "repository": {

diff --git a/example/lambdas/python-reference-task/package.json b/example/lambdas/python-reference-task/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-reference-task",
   "private": true,
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "Python reference task",
   "main": "index.js",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-task",

diff --git a/example/lambdas/s3AccessTest/package.json b/example/lambdas/s3AccessTest/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-s3-access",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "S3 Access Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/lambdas/snsS3Test/package.json b/example/lambdas/snsS3Test/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-sns-s3",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "SNS to S3 Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/lambdas/versionUpTest/package.json b/example/lambdas/versionUpTest/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-version-up",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "Version Up Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/package.json b/example/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/cumulus-integration-tests",
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "Cumulus Integration Test Deployment",
   "private": true,
   "main": "index.js",
@@ -44,32 +44,32 @@
     ]
   },
   "dependencies": {
-    "@cumulus/api": "15.0.3",
-    "@cumulus/api-client": "15.0.3",
-    "@cumulus/async-operations": "15.0.3",
-    "@cumulus/aws-client": "15.0.3",
-    "@cumulus/checksum": "15.0.3",
-    "@cumulus/cmr-client": "15.0.3",
-    "@cumulus/cmrjs": "15.0.3",
-    "@cumulus/common": "15.0.3",
-    "@cumulus/discover-granules": "15.0.3",
-    "@cumulus/discover-pdrs": "15.0.3",
-    "@cumulus/files-to-granules": "15.0.3",
-    "@cumulus/hello-world": "15.0.3",
-    "@cumulus/ingest": "15.0.3",
-    "@cumulus/integration-tests": "15.0.3",
-    "@cumulus/message": "15.0.3",
-    "@cumulus/move-granules": "15.0.3",
-    "@cumulus/parse-pdr": "15.0.3",
-    "@cumulus/pdr-status-check": "15.0.3",
-    "@cumulus/post-to-cmr": "15.0.3",
-    "@cumulus/queue-granules": "15.0.3",
-    "@cumulus/queue-pdrs": "15.0.3",
-    "@cumulus/sf-sqs-report": "15.0.3",
-    "@cumulus/sync-granule": "15.0.3",
-    "@cumulus/test-processing": "15.0.3"
+    "@cumulus/api": "15.0.4",
+    "@cumulus/api-client": "15.0.4",
+    "@cumulus/async-operations": "15.0.4",
+    "@cumulus/aws-client": "15.0.4",
+    "@cumulus/checksum": "15.0.4",
+    "@cumulus/cmr-client": "15.0.4",
+    "@cumulus/cmrjs": "15.0.4",
+    "@cumulus/common": "15.0.4",
+    "@cumulus/discover-granules": "15.0.4",
+    "@cumulus/discover-pdrs": "15.0.4",
+    "@cumulus/files-to-granules": "15.0.4",
+    "@cumulus/hello-world": "15.0.4",
+    "@cumulus/ingest": "15.0.4",
+    "@cumulus/integration-tests": "15.0.4",
+    "@cumulus/message": "15.0.4",
+    "@cumulus/move-granules": "15.0.4",
+    "@cumulus/parse-pdr": "15.0.4",
+    "@cumulus/pdr-status-check": "15.0.4",
+    "@cumulus/post-to-cmr": "15.0.4",
+    "@cumulus/queue-granules": "15.0.4",
+    "@cumulus/queue-pdrs": "15.0.4",
+    "@cumulus/sf-sqs-report": "15.0.4",
+    "@cumulus/sync-granule": "15.0.4",
+    "@cumulus/test-processing": "15.0.4"
   },
   "devDependencies": {
-    "@cumulus/test-data": "15.0.3"
+    "@cumulus/test-data": "15.0.4"
   }
 }
diff --git a/example/scripts/generate_ingest/package.json b/example/scripts/generate_ingest/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/generate_ingest",
   "private": true,
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "Script to generate test data for scaled ingest",
   "keywords": [
     "GIBS",
@@ -22,8 +22,8 @@
     "directory": "packages/types"
   },
   "dependencies": {
-    "@cumulus/aws-client": "15.0.3",
-    "@cumulus/common": "15.0.3"
+    "@cumulus/aws-client": "15.0.4",
+    "@cumulus/common": "15.0.4"
   },
   "author": "Cumulus Authors",
   "license": "Apache-2.0"

diff --git a/example/scripts/lib/package.json b/example/scripts/lib/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/example-lib",
   "private": true,
-  "version": "15.0.3",
+  "version": "15.0.4",
   "description": "example project libs",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/scripts/lib",
   "engines": {

diff --git a/example/scripts/run_test.sh b/example/scripts/run_test.sh
@@ -2,22 +2,26 @@
 
 set +e
 
-specName=$(echo "$2" | rev | cut -d'/' -f 1 | cut -d'.' -f 2 | rev)
+specName=$(echo "$3" | rev | cut -d'/' -f 1 | cut -d'.' -f 2 | rev)
 outputPath="${1}/${specName}-running.txt"
+testTimeout=$2
 
 TIMESTAMP=$(date "+%Y-%m-%dT%H:%M:%S")
-echo "$TIMESTAMP ../node_modules/.bin/jasmine $2 STARTED"
-
-../node_modules/.bin/jasmine --no-color "$2" > "$outputPath" 2>&1
+echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 STARTED"
+timeout "$testTimeout" ../node_modules/.bin/jasmine --no-color "$3" > "$outputPath" 2>&1
 result=$?
 
 TIMESTAMP=$(date "+%Y-%m-%dT%H:%M:%S")
 if [ "$result" -eq "0" ]; then
-  echo "$TIMESTAMP ../node_modules/.bin/jasmine $2 PASSED"
+  echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 PASSED"
   mv "$outputPath" "$1/${specName}-passed.txt"
+elif { [ "$result" -gt "124" ] && [ "$result" -lt "128" ] ;} || [ "$result" -eq "137" ]; then
+  echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 FAILED -- TIMEOUT"
+  mv "$outputPath" "$1/${specName}-failed.txt"
+  result=1
 else
-  echo "$TIMESTAMP ../node_modules/.bin/jasmine $2 FAILED"
+  echo "$TIMESTAMP ../node_modules/.bin/jasmine $3 FAILED"
   mv "$outputPath" "$1/${specName}-failed.txt"
 fi
 
-exit $result
+exit $result
diff --git a/example/scripts/tests-parallel.sh b/example/scripts/tests-parallel.sh
@@ -10,11 +10,12 @@ DOT_PID="$!"
 
 TESTS=$(find spec/parallel -type f -name '*spec.js' -or -name '*Spec.js')
 testOutputDir=scripts/test_output
+testTimeout=1200
 
 rm -r -f $testOutputDir
 mkdir -p $testOutputDir
 
-echo "" | ../node_modules/.bin/parallel -j "${INTEGRATION_CONCURRENCY:=0}" --timeout 1200 sh scripts/run_test.sh  $testOutputDir ::: $TESTS
+echo "" | ../node_modules/.bin/parallel -j "${INTEGRATION_CONCURRENCY:=0}" sh scripts/run_test.sh  $testOutputDir $testTimeout ::: $TESTS
 result=$?
 echo parallel tests complete: $result suite failures
 

diff --git a/example/spec/parallel/ingestGranule/IngestGranuleFailureSpec.js b/example/spec/parallel/ingestGranule/IngestGranuleFailureSpec.js
@@ -72,15 +72,22 @@ describe('The Ingest Granule failure workflow', () => {
       inputPayload = await setupTestGranuleForIngest(config.bucket, inputPayloadJson, granuleRegex, testSuffix, testDataFolder);
       pdrFilename = inputPayload.pdr.name;
 
-      // add a non-existent file to input payload to cause lambda error
+      // add a file with invalid schema (missing path field), and a non-existent file to input payload.
+      // .cmr.json is for testing retrieving cmr information when granule is failed
       inputPayload.granules[0].files = [
         {
-          name: 'non-existent-file',
-          key: 'non-existent-path/non-existent-file',
+          key: 'no-path-field/no-path-field-file',
           bucket: config.bucket,
+          name: 'no-path-field-file',
         },
+        {
+          name: 'non-existent-file.cmr.json',
+          path: 'non-existent-path',
+        },
+        ...inputPayload.granules[0].files,
       ];
 
+      console.log(`testSuffix: ${testSuffix}, granuleId: ${inputPayload.granules[0].granuleId}`);
       workflowExecution = await buildAndExecuteWorkflow(
         config.stackName,
         config.bucket,
@@ -91,6 +98,7 @@ describe('The Ingest Granule failure workflow', () => {
       );
     } catch (error) {
       beforeAllFailed = true;
+      console.log('IngestGranuleFailure beforeAll caught error', error);
       throw error;
     }
   });
@@ -219,7 +227,7 @@ describe('The Ingest Granule failure workflow', () => {
       expect(JSON.parse(execution.error.Cause)).toEqual(JSON.parse(syncGranFailedDetail.cause));
     });
 
-    it('fails the granule with an error object', async () => {
+    it('fails the granule with a list of errors', async () => {
       await waitForApiStatus(
         getGranule,
         {
@@ -235,8 +243,16 @@ describe('The Ingest Granule failure workflow', () => {
       });
 
       expect(granule.status).toBe('failed');
-      expect(granule.error.Error).toBeDefined();
-      expect(granule.error.Cause).toBeDefined();
+      console.log('IngestGranuleFailure granule.error', granule.error);
+      const errors = JSON.parse(granule.error.errors || []);
+      expect(errors.length).toBeGreaterThanOrEqual(2);
+      errors.forEach((error) => {
+        const isSchemaValidationError = (error.Error === 'CumulusMessageAdapterExecutionError') &&
+          error.Cause.includes('jsonschema.exceptions.ValidationError');
+        const isPostgresWriteError = error.Error.includes('Failed writing files to PostgreSQL') &&
+          error.Cause.includes('null value in column "bucket" violates not-null constraint');
+        expect(isSchemaValidationError || isPostgresWriteError).toBeTrue();
+      });
     });
   });
 });
diff --git a/example/spec/parallel/ingestGranule/IngestGranuleSuccessSpec.js b/example/spec/parallel/ingestGranule/IngestGranuleSuccessSpec.js
@@ -1275,7 +1275,6 @@ describe('The S3 Ingest Granules workflow', () => {
           await getGranule({
             prefix: config.stackName,
             granuleId: inputPayload.granules[0].granuleId,
-            expectedStatusCodes: 404,
           });
         } catch (error) {
           granuleResponseError = error;