Skip to content

Jk/cumulus 3135 fix integration tests #3403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jun 5, 2023
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- **CUMULUS-2985**
- Changed `onetime` rules RuleTrigger to only execute when the state is `ENABLED` and updated documentation to reflect the change
- Changed the `invokeRerun` function to only re-run enabled rules
- **CUMULUS-3315**
- Updated `@cumulus/api-client/granules.bulkOperation` to remove `ids`
parameter in favor of `granules` parameter, in the form of a
`@cumulus/types/ApiGranule` that requires the following keys: `[granuleId, collectionId]`

### Fixed

Expand Down
18 changes: 14 additions & 4 deletions example/spec/parallel/ingestGranule/IngestGranuleSuccessSpec.js
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ describe('The S3 Ingest Granules workflow', () => {

let beforeAllError;
let collection;
let collectionId;
let config;
let expectedPayload;
let expectedS3TagSet;
Expand All @@ -140,7 +141,7 @@ describe('The S3 Ingest Granules workflow', () => {
testDataFolder = createTestDataPath(testId);

collection = { name: `MOD09GQ${testSuffix}`, version: '006' };
const newCollectionId = constructCollectionId(collection.name, collection.version);
collectionId = constructCollectionId(collection.name, collection.version);
provider = { id: `s3_provider${testSuffix}` };

process.env.system_bucket = config.bucket;
Expand Down Expand Up @@ -193,7 +194,7 @@ describe('The S3 Ingest Granules workflow', () => {
},
});

expectedSyncGranulePayload = loadFileWithUpdatedGranuleIdPathAndCollection(templatedSyncGranuleFilename, granuleId, testDataFolder, newCollectionId, config.stackName);
expectedSyncGranulePayload = loadFileWithUpdatedGranuleIdPathAndCollection(templatedSyncGranuleFilename, granuleId, testDataFolder, collectionId, config.stackName);

expectedSyncGranulePayload.granules[0].dataType += testSuffix;
expectedSyncGranulePayload.granules[0].files[0].checksumType = inputPayload.granules[0].files[0].checksumType;
Expand Down Expand Up @@ -231,7 +232,7 @@ describe('The S3 Ingest Granules workflow', () => {
},
});

expectedPayload = loadFileWithUpdatedGranuleIdPathAndCollection(templatedOutputPayloadFilename, granuleId, testDataFolder, newCollectionId);
expectedPayload = loadFileWithUpdatedGranuleIdPathAndCollection(templatedOutputPayloadFilename, granuleId, testDataFolder, collectionId);
expectedPayload.granules[0].dataType += testSuffix;

// process.env.DISTRIBUTION_ENDPOINT needs to be set for below
Expand Down Expand Up @@ -932,7 +933,16 @@ describe('The S3 Ingest Granules workflow', () => {
bulkReingestResponse = await bulkReingestGranules({
prefix: config.stackName,
body: {
ids: [reingestGranuleId, fakeGranuleId],
granules: [
{
collectionId,
granuleId: reingestGranuleId,
},
{
collectionId,
granuleId: fakeGranuleId,
},
],
},
});
} catch (error) {
Expand Down
6 changes: 5 additions & 1 deletion example/spec/parallel/orca/OrcaBackupAndRecoverySpec.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const {
waitForCompletedExecution,
} = require('@cumulus/integration-tests');
const { LambdaStep } = require('@cumulus/integration-tests/sfnStep');
const { constructCollectionId } = require('@cumulus/message/Collections');

const { removeCollectionAndAllDependencies } = require('../../helpers/Collections');
const { buildAndStartWorkflow } = require('../../helpers/workflowUtils');
Expand Down Expand Up @@ -163,7 +164,10 @@ describe('The S3 Ingest Granules workflow', () => {

const response = await bulkOperation({
prefix: config.stackName,
ids: [granuleId],
granules: [{
granuleId,
collectionId: constructCollectionId(collection.name, collection.version),
}],
workflowName: recoveryWorkflowName,
});

Expand Down
12 changes: 6 additions & 6 deletions packages/api-client/src/granules.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import pRetry from 'p-retry';

import { ApiGranuleRecord, GranuleId, GranuleStatus } from '@cumulus/types/api/granules';
import { ApiGranuleRecord, ApiGranule, GranuleId, GranuleStatus } from '@cumulus/types/api/granules';
import { CollectionId } from '@cumulus/types/api/collections';
import Logger from '@cumulus/logger';

Expand Down Expand Up @@ -612,7 +612,7 @@ export const associateExecutionWithGranule = async (params: {
*/
export const bulkGranules = async (params: {
prefix: string,
body: object,
body: unknown,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
Expand Down Expand Up @@ -693,7 +693,7 @@ export const bulkReingestGranules = async (params: {
*
* @param {Object} params - params
* @param {string} params.prefix - the prefix configured for the stack
* @param {Array<string>} params.ids - the granules to have bulk operation on
* @param {Array<ApiGranuleRecord>} params.granules - the granules to have bulk operation on
* @param {string} params.workflowName - workflowName for the bulk operation execution
* @param {Function} params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
Expand All @@ -703,11 +703,11 @@ export const bulkReingestGranules = async (params: {
*/
export const bulkOperation = async (params: {
prefix: string,
ids: string[],
granules: ApiGranule[],
workflowName: string,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, ids, workflowName, callback = invokeApi } = params;
const { prefix, granules, workflowName, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
Expand All @@ -717,7 +717,7 @@ export const bulkOperation = async (params: {
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ ids, workflowName }),
body: JSON.stringify({ granules, workflowName }),
},
expectedStatusCodes: 202,
});
Expand Down
35 changes: 35 additions & 0 deletions packages/api-client/tests/test-granules.js
Original file line number Diff line number Diff line change
Expand Up @@ -531,3 +531,38 @@ test('associateExecutionWithGranule calls the callback with the expected object'
body,
}));
});

test('bulkOperation calls the callback with the expected object', async (t) => {
const workflowName = randomId('workflowName');
const granuleId = t.context.granuleId;
const collectionId = randomId('collectionId');
const granules = [{ granuleId, collectionId }];

const expected = {
prefix: t.context.testPrefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
path: '/granules/bulk/',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
granules,
workflowName,
}),
},
expectedStatusCodes: 202,
};

const callback = (configObject) => {
t.deepEqual(configObject, expected);
};

await t.notThrowsAsync(
granulesApi.bulkOperation({
callback,
prefix: t.context.testPrefix,
granules,
workflowName,
})
);
});