Skip to content
This repository was archived by the owner on Jun 28, 2022. It is now read-only.

Commit

Permalink
fix: pipeline bucket object duplication
Browse files Browse the repository at this point in the history
  • Loading branch information
arantespp committed Sep 18, 2021
1 parent 17b28b5 commit 3090748
Show file tree
Hide file tree
Showing 7 changed files with 254 additions and 23 deletions.
13 changes: 13 additions & 0 deletions packages/cli/src/deploy/cicd/cicd.template.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,13 @@
/* eslint-disable import/first */
import type { Pipeline } from './pipelines';

const projectName = 'my-project';

jest.mock('../../utils', () => ({
...(jest.requireActual('../../utils') as any),
getProjectName: jest.fn(() => projectName),
}));

import {
API_LOGICAL_ID,
PIPELINES_ARTIFACT_STORE_S3_BUCKET_LOGICAL_ID,
Expand All @@ -24,6 +33,10 @@ describe('testing CICD template', () => {
expect(template.Resources[API_LOGICAL_ID].Type).toEqual(
'AWS::Serverless::Api',
);
expect(
template.Resources.GitHubWebhooksApiV1ServerlessFunction.Properties
.Environment.Variables.TRIGGER_PIPELINES_OBJECT_KEY_PREFIX,
).toEqual(`cicd/pipelines/triggers/${projectName}`);
});
});

Expand Down
32 changes: 24 additions & 8 deletions packages/cli/src/deploy/cicd/cicd.template.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import { pascalCase } from 'change-case';
import yaml from 'js-yaml';

import { CloudFormationTemplate, getIamPath } from '../../utils';
import {
CloudFormationTemplate,
getIamPath,
getProjectName,
} from '../../utils';

import {
BASE_STACK_BUCKET_NAME_EXPORTED_NAME,
Expand All @@ -19,10 +23,7 @@ import {
PIPELINE_ECS_TASK_EXECUTION_STAGE_NAME,
} from './config';

import {
getTriggerPipelinesObjectKey,
TRIGGER_PIPELINES_OBJECT_KEY_PREFIX,
} from './getTriggerPipelineObjectKey';
import { getTriggerPipelinesObjectKey } from './getTriggerPipelineObjectKey';

export const API_LOGICAL_ID = 'ApiV1ServerlessApi';

Expand Down Expand Up @@ -244,6 +245,17 @@ export const getRepositoryImageBuilder = () => ({
},
});

/**
* This variable is used inside GitHub webhooks to identify the object key
* prefix of the file that triggers the pipelines.
*/
const triggerPipelinesObjectKeyPrefix = [
'cicd',
'pipelines',
'triggers',
getProjectName(),
].join('/');

export const getCicdTemplate = ({
pipelines = [],
cpu = ECS_TASK_DEFAULT_CPU,
Expand Down Expand Up @@ -484,7 +496,7 @@ export const getCicdTemplate = ({
Effect: 'Allow',
Resource: {
'Fn::Sub': [
`arn:aws:s3:::\${BucketName}/${TRIGGER_PIPELINES_OBJECT_KEY_PREFIX}*`,
`arn:aws:s3:::\${BucketName}/${triggerPipelinesObjectKeyPrefix}*`,
{
BucketName: {
'Fn::ImportValue': BASE_STACK_BUCKET_NAME_EXPORTED_NAME,
Expand Down Expand Up @@ -578,6 +590,7 @@ export const getCicdTemplate = ({
BASE_STACK_BUCKET_NAME: {
'Fn::ImportValue': BASE_STACK_BUCKET_NAME_EXPORTED_NAME,
},
TRIGGER_PIPELINES_OBJECT_KEY_PREFIX: triggerPipelinesObjectKeyPrefix,
PIPELINES_JSON: JSON.stringify(pipelines),
...executeEcsTaskVariables,
},
Expand Down Expand Up @@ -837,7 +850,7 @@ export const getCicdTemplate = ({
Action: 's3:*',
Resource: {
'Fn::Sub': [
`arn:aws:s3:::\${BucketName}/${TRIGGER_PIPELINES_OBJECT_KEY_PREFIX}*`,
`arn:aws:s3:::\${BucketName}/${triggerPipelinesObjectKeyPrefix}*`,
{
BucketName: {
'Fn::ImportValue': BASE_STACK_BUCKET_NAME_EXPORTED_NAME,
Expand Down Expand Up @@ -897,7 +910,10 @@ export const getCicdTemplate = ({
S3Bucket: {
'Fn::ImportValue': BASE_STACK_BUCKET_NAME_EXPORTED_NAME,
},
S3ObjectKey: getTriggerPipelinesObjectKey(pipeline),
S3ObjectKey: getTriggerPipelinesObjectKey({
prefix: triggerPipelinesObjectKeyPrefix,
pipeline,
}),
},
Name: `Pipeline${pipelinePascalCase}S3SourceAction`,
OutputArtifacts: [
Expand Down
13 changes: 8 additions & 5 deletions packages/cli/src/deploy/cicd/getTriggerPipelineObjectKey.spec.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import * as faker from 'faker';

import { getTriggerPipelinesObjectKey } from './getTriggerPipelineObjectKey';
import { Pipeline } from './pipelines';

test('main pipeline', () => {
expect(getTriggerPipelinesObjectKey('main')).toContain('/main.zip');
});
const prefix = faker.random.word();

test('tag pipeline', () => {
expect(getTriggerPipelinesObjectKey('tag')).toContain('/tag.zip');
test.each<[Pipeline]>([['tag'], ['main']])('main pipeline', (pipeline) => {
expect(getTriggerPipelinesObjectKey({ prefix, pipeline })).toContain(
`${prefix}/${pipeline}.zip`,
);
});
12 changes: 8 additions & 4 deletions packages/cli/src/deploy/cicd/getTriggerPipelineObjectKey.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
import type { Pipeline } from './pipelines';

export const TRIGGER_PIPELINES_OBJECT_KEY_PREFIX = 'cicd/pipelines/triggers/';

/**
* The file with this key inside the source S3 key of main and tag pipelines
* will trigger those pipelines.
*/
export const getTriggerPipelinesObjectKey = (pipeline: Pipeline) => {
return `${TRIGGER_PIPELINES_OBJECT_KEY_PREFIX}${pipeline}.zip`;
export const getTriggerPipelinesObjectKey = ({
prefix,
pipeline,
}: {
prefix: string;
pipeline: Pipeline;
}) => {
return `${prefix}/${pipeline}.zip`;
};
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ export const getProcessEnvVariable = (env: string): string => {
return process.env[env] as string;
}

throw new Error(`process.env.${env} doesn't exist.`);
throw new Error(`process.env.${env} is not defined.`);
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
/* eslint-disable import/first */

// const webhooksOnErrorMock = jest.fn();

// const webhooksReceiveMock = jest.fn();

// jest.mock('@octokit/webhooks', () => ({
// Webhooks: jest.fn().mockReturnValue({
// onError: webhooksOnErrorMock,
// receive: webhooksReceiveMock,
// }),
// }));

const putObjectMock = jest.fn().mockReturnValue({ promise: jest.fn() });

jest.mock('aws-sdk', () => ({
ECS: jest.fn(),
S3: jest.fn().mockReturnValue({
putObject: putObjectMock,
}),
}));

import {
githubWebhooksApiV1Handler,
webhooks,
} from './githubWebhooksApiV1.handler';

const context = {} as any;

const callback = jest.fn();

const handler = (event: any) =>
githubWebhooksApiV1Handler(event, context, callback);

const xGitHubDelivery = 'xGitHubDelivery';

const xGitHubEvent = 'xGitHubEvent';

const xHubSignature = 'xHubSignature';

const webhooksReceiveMock = jest.spyOn(webhooks, 'receive');

beforeEach(() => {
delete process.env.PIPELINES_JSON;
delete process.env.TRIGGER_PIPELINES_OBJECT_KEY_PREFIX;
delete process.env.BASE_STACK_BUCKET_NAME;

webhooksReceiveMock.mockClear();
});

test('should call S3 putObject', async () => {
process.env.PIPELINES_JSON = JSON.stringify(['main']);
process.env.TRIGGER_PIPELINES_OBJECT_KEY_PREFIX = 'some/prefix';
process.env.BASE_STACK_BUCKET_NAME = 'base-stack';

const body = { ref: 'refs/heads/main' };

const response: any = await handler({
headers: {
'X-GitHub-Delivery': xGitHubDelivery,
'X-GitHub-Event': 'push',
'X-Hub-Signature-256': xHubSignature,
},
body: JSON.stringify(body),
});

expect(putObjectMock).toHaveBeenCalledWith(
expect.objectContaining({
Body: expect.any(Buffer),
Bucket: 'base-stack',
Key: 'some/prefix/main.zip',
}),
);

expect(response).toMatchObject({ body: '{"ok":true}', statusCode: 200 });
});

test('should throw process.env.BASE_STACK_BUCKET_NAME is not defined', async () => {
process.env.PIPELINES_JSON = JSON.stringify(['main']);
process.env.TRIGGER_PIPELINES_OBJECT_KEY_PREFIX = 'some/prefix';

const body = { ref: 'refs/heads/main' };

const response: any = await handler({
headers: {
'X-GitHub-Delivery': xGitHubDelivery,
'X-GitHub-Event': 'push',
'X-Hub-Signature-256': xHubSignature,
},
body: JSON.stringify(body),
});

expect(webhooksReceiveMock).toHaveBeenCalledWith({
id: xGitHubDelivery,
name: 'push',
payload: body,
});

expect(response.statusCode).toBe(500);
expect(response.body).toContain(
'process.env.BASE_STACK_BUCKET_NAME is not defined.',
);
});

test('should throw process.env.TRIGGER_PIPELINES_OBJECT_KEY_PREFIX is not defined', async () => {
process.env.PIPELINES_JSON = JSON.stringify(['main']);

const body = { ref: 'refs/heads/main' };

const response: any = await handler({
headers: {
'X-GitHub-Delivery': xGitHubDelivery,
'X-GitHub-Event': 'push',
'X-Hub-Signature-256': xHubSignature,
},
body: JSON.stringify(body),
});

expect(webhooksReceiveMock).toHaveBeenCalledWith({
id: xGitHubDelivery,
name: 'push',
payload: body,
});

expect(response.statusCode).toBe(500);
expect(response.body).toContain(
'process.env.TRIGGER_PIPELINES_OBJECT_KEY_PREFIX is not defined',
);
});

test('should call webhook.receive properly', async () => {
const body = { someProperty: 'someValue' };

const response = await handler({
headers: {
'X-GitHub-Delivery': xGitHubDelivery,
'X-GitHub-Event': xGitHubEvent,
'X-Hub-Signature-256': xHubSignature,
},
body: JSON.stringify(body),
});

expect(webhooksReceiveMock).toHaveBeenCalledWith({
id: xGitHubDelivery,
name: xGitHubEvent,
payload: body,
});
expect(response).toMatchObject({ body: '{"ok":true}', statusCode: 200 });
});

test("should return event.body doesn't exist", async () => {
const response = await handler({});
expect(response).toEqual({
body: "event.body doesn't exist.",
statusCode: 500,
});
});

test("should return X-GitHub-Delivery doesn't exist", async () => {
const response = await handler({ headers: {}, body: '{}' });
expect(response).toEqual({
body: "X-GitHub-Delivery doesn't exist.",
statusCode: 500,
});
});

test("should return X-GitHub-Event doesn't exist", async () => {
const response = await handler({
headers: { 'X-GitHub-Delivery': xGitHubDelivery },
body: '{}',
});
expect(response).toEqual({
body: "X-GitHub-Event doesn't exist.",
statusCode: 500,
});
});

test("should return X-Hub-Signature-256 or X-Hub-Signature doesn't exist", async () => {
const response = await handler({
headers: {
'X-GitHub-Delivery': xGitHubDelivery,
'X-GitHub-Event': xGitHubEvent,
},
body: '{}',
});
expect(response).toEqual({
body: "X-Hub-Signature-256 or X-Hub-Signature doesn't exist.",
statusCode: 500,
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ import { getProcessEnvVariable } from './getProcessEnvVariable';

const s3 = new S3();

/**
* Put outside of the handler to be able to spy on it.
*/
export const webhooks = new Webhooks({ secret: '123' });

/**
* When this file is saved on S3, a CodePipeline pipeline is started.
*/
Expand All @@ -21,6 +26,8 @@ const putJobDetails = async ({
pipeline: Pipeline;
details: any;
}) => {
const prefix = getProcessEnvVariable('TRIGGER_PIPELINES_OBJECT_KEY_PREFIX');

const zip = new AdmZip();

const content = JSON.stringify(details);
Expand All @@ -31,7 +38,7 @@ const putJobDetails = async ({
.putObject({
Body: zip.toBuffer(),
Bucket: getProcessEnvVariable('BASE_STACK_BUCKET_NAME'),
Key: getTriggerPipelinesObjectKey(pipeline),
Key: getTriggerPipelinesObjectKey({ prefix, pipeline }),
})
.promise();
};
Expand Down Expand Up @@ -69,8 +76,6 @@ export const githubWebhooksApiV1Handler: ProxyHandler = async (
throw new Error("X-Hub-Signature-256 or X-Hub-Signature doesn't exist.");
}

const webhooks = new Webhooks({ secret: '123' });

const pipelines: Pipeline[] = JSON.parse(
process.env.PIPELINES_JSON || JSON.stringify([]),
);
Expand Down Expand Up @@ -107,7 +112,9 @@ export const githubWebhooksApiV1Handler: ProxyHandler = async (
});
},
);
}

if (pipelines.includes('closed-pr')) {
webhooks.on(['pull_request.closed'], async ({ payload }) => {
/**
* https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
Expand Down Expand Up @@ -151,8 +158,6 @@ export const githubWebhooksApiV1Handler: ProxyHandler = async (
}

webhooks.onError((onErrorEvent) => {
console.error('Webhooks on error.');
console.error(JSON.stringify(onErrorEvent, null, 2));
throw onErrorEvent;
});

Expand Down

0 comments on commit 3090748

Please sign in to comment.