From 3df6774d6f13ecea465698743856a169f1de749f Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Thu, 15 Apr 2021 17:05:47 -0400 Subject: [PATCH 1/4] [rush-lib] improve support for S3 storage for the buildCache --- .../buildCache/AmazonS3/AmazonS3Client.ts | 46 +++++++++++++++---- .../@microsoft/rush/s3_2021-04-15-21-04.json | 11 +++++ 2 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 common/changes/@microsoft/rush/s3_2021-04-15-21-04.json diff --git a/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts b/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts index 275d80e472e..a065750abfc 100644 --- a/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts +++ b/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts @@ -10,10 +10,14 @@ import { IPutFetchOptions, IGetFetchOptions, WebClient } from '../../../utilitie const CONTENT_HASH_HEADER_NAME: 'x-amz-content-sha256' = 'x-amz-content-sha256'; const DATE_HEADER_NAME: 'x-amz-date' = 'x-amz-date'; const HOST_HEADER_NAME: 'host' = 'host'; +const SECURITY_TOKEN_HEADER_NAME: 'x-amz-security-token' = 'x-amz-security-token'; + +const DEFAULT_S3_REGION: 'us-east-1' = 'us-east-1'; export interface IAmazonS3Credentials { accessKeyId: string; secretAccessKey: string; + sessionToken: string | undefined; } interface IIsoDateString { @@ -49,14 +53,15 @@ export class AmazonS3Client { return undefined; } - const splitIndex: number = credentialString.indexOf(':'); - if (splitIndex === -1) { + const fields: string[] = credentialString.split(':'); + if (fields.length < 2 || fields.length > 3) { throw new Error('Amazon S3 credential is in an unexpected format.'); } return { - accessKeyId: credentialString.substring(0, splitIndex), - secretAccessKey: credentialString.substring(splitIndex + 1) + accessKeyId: fields[0], + secretAccessKey: fields[1], + sessionToken: fields[2] }; } @@ -91,15 +96,26 @@ export class AmazonS3Client { ): Promise { const isoDateString: IIsoDateString = this._getIsoDateString(); const bodyHash: string = this._getSha256(body); - const host: string = `${this._s3Bucket}.s3.amazonaws.com`; - + const host: string = this._getHost(); const headers: fetch.Headers = new fetch.Headers(); headers.set(DATE_HEADER_NAME, isoDateString.dateTime); headers.set(CONTENT_HASH_HEADER_NAME, bodyHash); if (this._credentials) { // Compute the authorization header. See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - const signedHeaderNames: string = `${HOST_HEADER_NAME};${CONTENT_HASH_HEADER_NAME};${DATE_HEADER_NAME}`; + const signedHeaderNames: string[] = [HOST_HEADER_NAME, CONTENT_HASH_HEADER_NAME, DATE_HEADER_NAME]; + const canonicalHeaders: string[] = [ + `${HOST_HEADER_NAME}:${host}`, + `${CONTENT_HASH_HEADER_NAME}:${bodyHash}`, + `${DATE_HEADER_NAME}:${isoDateString.dateTime}` + ]; + + // Handle signing with temporary credentials (via sts:assume-role) + if (this._credentials.sessionToken) { + signedHeaderNames.push(SECURITY_TOKEN_HEADER_NAME); + canonicalHeaders.push(`${SECURITY_TOKEN_HEADER_NAME}:${this._credentials.sessionToken}`); + } + // The canonical request looks like this: // GET // /test.txt @@ -115,9 +131,7 @@ export class AmazonS3Client { verb, `/${objectName}`, '', // we don't use query strings for these requests - `${HOST_HEADER_NAME}:${host}`, - `${CONTENT_HASH_HEADER_NAME}:${bodyHash}`, - `${DATE_HEADER_NAME}:${isoDateString.dateTime}`, + ...canonicalHeaders, '', signedHeaderNames, bodyHash @@ -149,6 +163,10 @@ export class AmazonS3Client { const authorizationHeader: string = `AWS4-HMAC-SHA256 Credential=${this._credentials.accessKeyId}/${scope},SignedHeaders=${signedHeaderNames},Signature=${signature}`; headers.set('Authorization', authorizationHeader); + if (this._credentials.sessionToken) { + // Handle signing with temporary credentials (via sts:assume-role) + headers.set('X-Amz-Security-Token', this._credentials.sessionToken); + } } const webFetchOptions: IGetFetchOptions | IPutFetchOptions = { @@ -207,6 +225,14 @@ export class AmazonS3Client { throw new Error(`Amazon S3 responded with status code ${response.status} (${response.statusText})`); } + private _getHost(): string { + if (this._s3Region === DEFAULT_S3_REGION) { + return `${this._s3Bucket}.s3.amazonaws.com`; + } else { + return `${this._s3Bucket}.s3-${this._s3Region}.amazonaws.com`; + } + } + /** * Validates a S3 bucket name. * {@link https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html} diff --git a/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json b/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json new file mode 100644 index 00000000000..25b4df050a5 --- /dev/null +++ b/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "The build cache can now use buckets outside the default region", + "type": "none" + } + ], + "packageName": "@microsoft/rush", + "email": "nelson.work@gmail.com" +} \ No newline at end of file From 259cb8e1fbc1b1f99426417c154d34caa3340788 Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Thu, 15 Apr 2021 17:18:54 -0400 Subject: [PATCH 2/4] [rush-lib] fix signed header names --- apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts b/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts index a065750abfc..08c62b737dc 100644 --- a/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts +++ b/apps/rush-lib/src/logic/buildCache/AmazonS3/AmazonS3Client.ts @@ -133,7 +133,7 @@ export class AmazonS3Client { '', // we don't use query strings for these requests ...canonicalHeaders, '', - signedHeaderNames, + signedHeaderNames.join(';'), bodyHash ].join('\n'); const canonicalRequestHash: string = this._getSha256(canonicalRequest); From 20cd77827d87181542636758ea9ac4b2f8858139 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Fri, 16 Apr 2021 19:20:20 -0700 Subject: [PATCH 3/4] Add Amazon S3 request tests --- .../AmazonS3/test/AmazonS3Client.test.ts | 311 ++++++++- .../__snapshots__/AmazonS3Client.test.ts.snap | 607 ++++++++++++++++++ 2 files changed, 902 insertions(+), 16 deletions(-) diff --git a/apps/rush-lib/src/logic/buildCache/AmazonS3/test/AmazonS3Client.test.ts b/apps/rush-lib/src/logic/buildCache/AmazonS3/test/AmazonS3Client.test.ts index 3ba01f7e90a..160695d5a40 100644 --- a/apps/rush-lib/src/logic/buildCache/AmazonS3/test/AmazonS3Client.test.ts +++ b/apps/rush-lib/src/logic/buildCache/AmazonS3/test/AmazonS3Client.test.ts @@ -1,67 +1,93 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { Response, ResponseInit } from 'node-fetch'; + import { IAmazonS3BuildCacheProviderOptions } from '../AmazonS3BuildCacheProvider'; -import { AmazonS3Client } from '../AmazonS3Client'; +import { AmazonS3Client, IAmazonS3Credentials } from '../AmazonS3Client'; +import { WebClient } from '../../../../utilities/WebClient'; -const DUMMY_OPTIONS: Omit = { +const DUMMY_OPTIONS_WITHOUT_BUCKET: Omit = { s3Region: 'us-east-1', isCacheWriteAllowed: true }; +const DUMMY_OPTIONS: IAmazonS3BuildCacheProviderOptions = { + ...DUMMY_OPTIONS_WITHOUT_BUCKET, + s3Bucket: 'test-s3-bucket' +}; + +class MockedDate extends Date { + public constructor() { + super(2020, 3, 18, 12, 32, 42, 493); + } + + public toISOString(): string { + return '2020-04-18T12:32:42.493Z'; + } +} + describe('AmazonS3Client', () => { it('Rejects invalid S3 bucket names', () => { expect( - () => new AmazonS3Client(undefined, { s3Bucket: undefined!, ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: undefined!, ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: '-abc', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: '-abc', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: 'a!bc', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: 'a!bc', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: 'a', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: 'a', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: '10.10.10.10', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: '10.10.10.10', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: 'abc..d', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: 'abc..d', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: 'abc.-d', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: 'abc.-d', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: 'abc-.d', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: 'abc-.d', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); expect( - () => new AmazonS3Client(undefined, { s3Bucket: 'abc-', ...DUMMY_OPTIONS }) + () => new AmazonS3Client(undefined, { s3Bucket: 'abc-', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) ).toThrowErrorMatchingSnapshot(); }); it('Accepts valid S3 bucket names', () => { - expect(() => new AmazonS3Client(undefined, { s3Bucket: 'abc123', ...DUMMY_OPTIONS })).not.toThrow(); + expect( + () => new AmazonS3Client(undefined, { s3Bucket: 'abc123', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) + ).not.toThrow(); - expect(() => new AmazonS3Client(undefined, { s3Bucket: 'abc', ...DUMMY_OPTIONS })).not.toThrow(); + expect( + () => new AmazonS3Client(undefined, { s3Bucket: 'abc', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) + ).not.toThrow(); - expect(() => new AmazonS3Client(undefined, { s3Bucket: 'foo-bar-baz', ...DUMMY_OPTIONS })).not.toThrow(); + expect( + () => new AmazonS3Client(undefined, { s3Bucket: 'foo-bar-baz', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) + ).not.toThrow(); - expect(() => new AmazonS3Client(undefined, { s3Bucket: 'foo.bar.baz', ...DUMMY_OPTIONS })).not.toThrow(); + expect( + () => new AmazonS3Client(undefined, { s3Bucket: 'foo.bar.baz', ...DUMMY_OPTIONS_WITHOUT_BUCKET }) + ).not.toThrow(); }); it('Does not allow upload without credentials', async () => { const client: AmazonS3Client = new AmazonS3Client(undefined, { s3Bucket: 'foo.bar.baz', - ...DUMMY_OPTIONS + ...DUMMY_OPTIONS_WITHOUT_BUCKET }); try { await client.uploadObjectAsync('temp', undefined!); @@ -70,4 +96,257 @@ describe('AmazonS3Client', () => { expect(e).toMatchSnapshot(); } }); + + describe('Making requests', () => { + interface IResponseOptions { + body?: string; + responseInit: ResponseInit; + } + + let realDate: typeof Date; + beforeEach(() => { + realDate = global.Date; + global.Date = MockedDate as typeof Date; + }); + + afterEach(() => { + jest.restoreAllMocks(); + global.Date = realDate; + }); + + async function makeS3ClientRequestAsync( + credentials: IAmazonS3Credentials | undefined, + options: IAmazonS3BuildCacheProviderOptions, + request: (s3Client: AmazonS3Client) => Promise, + response: IResponseOptions + ): Promise { + const spy: jest.SpyInstance = jest + .spyOn(WebClient.prototype, 'fetchAsync') + .mockReturnValue(Promise.resolve(new Response(response.body, response.responseInit))); + + const s3Client: AmazonS3Client = new AmazonS3Client(credentials, options); + let result: TResponse; + let error: Error | undefined; + try { + result = await request(s3Client); + } catch (e) { + error = e; + } + + expect(spy).toHaveBeenCalledTimes(1); + expect(spy.mock.calls[0]).toMatchSnapshot(); + + if (error) { + throw error; + } else { + return result!; + } + } + + async function runAndExpectErrorAsync(fnAsync: () => Promise): Promise { + try { + await fnAsync(); + fail('Expected an error to be thrown'); + } catch (e) { + expect(e).toMatchSnapshot(); + } + } + + describe('Getting an object', () => { + async function makeGetRequestAsync( + credentials: IAmazonS3Credentials | undefined, + options: IAmazonS3BuildCacheProviderOptions, + objectName: string, + response: IResponseOptions + ): Promise { + return await makeS3ClientRequestAsync( + credentials, + options, + async (s3Client) => { + return await s3Client.getObjectAsync(objectName); + }, + response + ); + } + + function registerGetTests(credentials: IAmazonS3Credentials | undefined): void { + it('Can get an object', async () => { + const expectedContents: string = 'abc123-contents'; + + const result: Buffer | undefined = await makeGetRequestAsync(credentials, DUMMY_OPTIONS, 'abc123', { + body: expectedContents, + responseInit: { + status: 200 + } + }); + expect(result).toBeDefined(); + expect(result?.toString()).toBe(expectedContents); + }); + + it('Can get an object from a different region', async () => { + const expectedContents: string = 'abc123-contents'; + + const result: Buffer | undefined = await makeGetRequestAsync( + credentials, + { ...DUMMY_OPTIONS, s3Region: 'us-west-1' }, + 'abc123', + { + body: expectedContents, + responseInit: { + status: 200 + } + } + ); + expect(result).toBeDefined(); + expect(result?.toString()).toBe(expectedContents); + }); + + it('Handles a missing object', async () => { + const result: Buffer | undefined = await makeGetRequestAsync(credentials, DUMMY_OPTIONS, 'abc123', { + responseInit: { + status: 404, + statusText: 'Not Found' + } + }); + expect(result).toBeUndefined(); + }); + + it('Handles an unexpected error', async () => { + await runAndExpectErrorAsync( + async () => + await makeGetRequestAsync(credentials, DUMMY_OPTIONS, 'abc123', { + responseInit: { + status: 500, + statusText: 'Server Error' + } + }) + ); + }); + } + + describe('Without credentials', () => { + registerGetTests(undefined); + + it('Handles missing credentials object', async () => { + const result: Buffer | undefined = await makeGetRequestAsync(undefined, DUMMY_OPTIONS, 'abc123', { + responseInit: { + status: 403, + statusText: 'Unauthorized' + } + }); + expect(result).toBeUndefined(); + }); + }); + + function registerGetWithCredentialsTests(credentials: IAmazonS3Credentials): void { + registerGetTests(credentials); + + it('Handles a 403 error', async () => { + await runAndExpectErrorAsync( + async () => + await makeGetRequestAsync(credentials, DUMMY_OPTIONS, 'abc123', { + responseInit: { + status: 403, + statusText: 'Unauthorized' + } + }) + ); + }); + } + + describe('With credentials', () => { + registerGetWithCredentialsTests({ + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + sessionToken: undefined + }); + }); + + describe('With credentials including a session token', () => { + registerGetWithCredentialsTests({ + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + sessionToken: 'sessionToken' + }); + }); + }); + + describe('Uploading an object', () => { + async function makeUploadRequestAsync( + credentials: IAmazonS3Credentials | undefined, + options: IAmazonS3BuildCacheProviderOptions, + objectName: string, + objectContents: string, + response: IResponseOptions + ): Promise { + return await makeS3ClientRequestAsync( + credentials, + options, + async (s3Client) => { + return await s3Client.uploadObjectAsync(objectName, Buffer.from(objectContents)); + }, + response + ); + } + + it('Throws an error if credentials are not provided', async () => { + await runAndExpectErrorAsync( + async () => + await makeUploadRequestAsync(undefined, DUMMY_OPTIONS, 'abc123', 'abc123-contents', undefined!) + ); + }); + + function registerUploadTests(credentials: IAmazonS3Credentials): void { + it('Uploads an object', async () => { + await makeUploadRequestAsync(credentials, DUMMY_OPTIONS, 'abc123', 'abc123-contents', { + responseInit: { + status: 200 + } + }); + }); + + it('Uploads an object to a different region', async () => { + await makeUploadRequestAsync( + credentials, + { ...DUMMY_OPTIONS, s3Region: 'us-west-1' }, + 'abc123', + 'abc123-contents', + { + responseInit: { + status: 200 + } + } + ); + }); + + it('Handles an unexpected error code', async () => { + await runAndExpectErrorAsync( + async () => + await makeUploadRequestAsync(credentials, DUMMY_OPTIONS, 'abc123', 'abc123-contents', { + responseInit: { + status: 500, + statusText: 'Server Error' + } + }) + ); + }); + } + + describe('With credentials', () => { + registerUploadTests({ + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + sessionToken: undefined + }); + }); + + describe('With credentials including a session token', () => { + registerUploadTests({ + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + sessionToken: 'sessionToken' + }); + }); + }); + }); }); diff --git a/apps/rush-lib/src/logic/buildCache/AmazonS3/test/__snapshots__/AmazonS3Client.test.ts.snap b/apps/rush-lib/src/logic/buildCache/AmazonS3/test/__snapshots__/AmazonS3Client.test.ts.snap index 1744ac182e7..e14fa6a2b69 100644 --- a/apps/rush-lib/src/logic/buildCache/AmazonS3/test/__snapshots__/AmazonS3Client.test.ts.snap +++ b/apps/rush-lib/src/logic/buildCache/AmazonS3/test/__snapshots__/AmazonS3Client.test.ts.snap @@ -2,6 +2,613 @@ exports[`AmazonS3Client Does not allow upload without credentials 1`] = `[Error: Credentials are required to upload objects to S3.]`; +exports[`AmazonS3Client Making requests Getting an object With credentials Can get an object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=11441edef046611ecf352daa2bcae55584d302a31b3390ee865781671caf791a", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials Can get an object from a different region 1`] = ` +Array [ + "https://test-s3-bucket.s3-us-west-1.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=19d94ed314002214315e8e9816ca31c97e7c834f7494c3c61046550f12358c21", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials Handles a 403 error 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=11441edef046611ecf352daa2bcae55584d302a31b3390ee865781671caf791a", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials Handles a 403 error 2`] = `[Error: Amazon S3 responded with status code 403 (Unauthorized)]`; + +exports[`AmazonS3Client Making requests Getting an object With credentials Handles a missing object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=11441edef046611ecf352daa2bcae55584d302a31b3390ee865781671caf791a", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials Handles an unexpected error 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=11441edef046611ecf352daa2bcae55584d302a31b3390ee865781671caf791a", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials Handles an unexpected error 2`] = `[Error: Amazon S3 responded with status code 500 (Server Error)]`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Can get an object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=242129cdc7470382920680b887e5899a56eb94103e11ef9b96a45ee0d2bff5c7", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Can get an object from a different region 1`] = ` +Array [ + "https://test-s3-bucket.s3-us-west-1.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=b3f43b86838b915e38f9900e5049870ca53db5792b578403f2d46185cc6bb3f1", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Handles a 403 error 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=242129cdc7470382920680b887e5899a56eb94103e11ef9b96a45ee0d2bff5c7", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Handles a 403 error 2`] = `[Error: Amazon S3 responded with status code 403 (Unauthorized)]`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Handles a missing object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=242129cdc7470382920680b887e5899a56eb94103e11ef9b96a45ee0d2bff5c7", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Handles an unexpected error 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=242129cdc7470382920680b887e5899a56eb94103e11ef9b96a45ee0d2bff5c7", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object With credentials including a session token Handles an unexpected error 2`] = `[Error: Amazon S3 responded with status code 500 (Server Error)]`; + +exports[`AmazonS3Client Making requests Getting an object Without credentials Can get an object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object Without credentials Can get an object from a different region 1`] = ` +Array [ + "https://test-s3-bucket.s3-us-west-1.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object Without credentials Handles a missing object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object Without credentials Handles an unexpected error 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Getting an object Without credentials Handles an unexpected error 2`] = `[Error: Amazon S3 responded with status code 500 (Server Error)]`; + +exports[`AmazonS3Client Making requests Getting an object Without credentials Handles missing credentials object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "headers": Headers { + Symbol(map): Object { + "x-amz-content-sha256": Array [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "GET", + }, +] +`; + +exports[`AmazonS3Client Making requests Uploading an object Throws an error if credentials are not provided 1`] = `[TypeError: Cannot read property 'body' of undefined]`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials Handles an unexpected error code 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "body": Object { + "data": Array [ + 97, + 98, + 99, + 49, + 50, + 51, + 45, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 115, + ], + "type": "Buffer", + }, + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=1db5024ed7d91ac512762a2c70490754def64dc5ed61e3e98d090233ebe0f79c", + ], + "x-amz-content-sha256": Array [ + "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "PUT", + }, +] +`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials Handles an unexpected error code 2`] = `[Error: Amazon S3 responded with status code 500 (Server Error)]`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials Uploads an object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "body": Object { + "data": Array [ + 97, + 98, + 99, + 49, + 50, + 51, + 45, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 115, + ], + "type": "Buffer", + }, + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=1db5024ed7d91ac512762a2c70490754def64dc5ed61e3e98d090233ebe0f79c", + ], + "x-amz-content-sha256": Array [ + "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "PUT", + }, +] +`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials Uploads an object to a different region 1`] = ` +Array [ + "https://test-s3-bucket.s3-us-west-1.amazonaws.com/abc123", + Object { + "body": Object { + "data": Array [ + 97, + 98, + 99, + 49, + 50, + 51, + 45, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 115, + ], + "type": "Buffer", + }, + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,Signature=35a4edef214657ec5799666681f637951d01b3cbf9ec3754f858ce8b722c026c", + ], + "x-amz-content-sha256": Array [ + "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "PUT", + }, +] +`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials including a session token Handles an unexpected error code 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "body": Object { + "data": Array [ + 97, + 98, + 99, + 49, + 50, + 51, + 45, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 115, + ], + "type": "Buffer", + }, + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=f50f9b3a7b33b58809a8da7216b68ca8730fd157cc7aef4c945fa5df1a22cd03", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "PUT", + }, +] +`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials including a session token Handles an unexpected error code 2`] = `[Error: Amazon S3 responded with status code 500 (Server Error)]`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials including a session token Uploads an object 1`] = ` +Array [ + "https://test-s3-bucket.s3.amazonaws.com/abc123", + Object { + "body": Object { + "data": Array [ + 97, + 98, + 99, + 49, + 50, + 51, + 45, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 115, + ], + "type": "Buffer", + }, + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-east-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=f50f9b3a7b33b58809a8da7216b68ca8730fd157cc7aef4c945fa5df1a22cd03", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "PUT", + }, +] +`; + +exports[`AmazonS3Client Making requests Uploading an object With credentials including a session token Uploads an object to a different region 1`] = ` +Array [ + "https://test-s3-bucket.s3-us-west-1.amazonaws.com/abc123", + Object { + "body": Object { + "data": Array [ + 97, + 98, + 99, + 49, + 50, + 51, + 45, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 115, + ], + "type": "Buffer", + }, + "headers": Headers { + Symbol(map): Object { + "Authorization": Array [ + "AWS4-HMAC-SHA256 Credential=accessKeyId/20200418/us-west-1/s3/aws4_request,SignedHeaders=host,x-amz-content-sha256,x-amz-date,x-amz-security-token,Signature=e846064053af5730311f5a8dd565139c9fdc9de4f9d1c12b8f2f77f619b7d2e1", + ], + "X-Amz-Security-Token": Array [ + "sessionToken", + ], + "x-amz-content-sha256": Array [ + "f8e4bdb2ca9c0f90b0fe56e32bf509ba44b73e2f52af123832f9ddbfe7e8fafa", + ], + "x-amz-date": Array [ + "20200418T123242Z", + ], + }, + }, + "verb": "PUT", + }, +] +`; + exports[`AmazonS3Client Rejects invalid S3 bucket names 1`] = `"A S3 bucket name must be provided"`; exports[`AmazonS3Client Rejects invalid S3 bucket names 2`] = `"The bucket name \\"-abc\\" is invalid. A S3 bucket name must start with a lowercase alphanumerical character."`; From aaa9d527f181b8ba8d98e5f7d94fd34f4658cc15 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Fri, 16 Apr 2021 19:22:39 -0700 Subject: [PATCH 4/4] Update changelog. --- common/changes/@microsoft/rush/s3_2021-04-15-21-04.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json b/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json index 25b4df050a5..0a208eeb159 100644 --- a/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json +++ b/common/changes/@microsoft/rush/s3_2021-04-15-21-04.json @@ -2,10 +2,10 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "The build cache can now use buckets outside the default region", + "comment": "The Amazon S3 build cloud cache provider can now use buckets outside the default region", "type": "none" } ], "packageName": "@microsoft/rush", "email": "nelson.work@gmail.com" -} \ No newline at end of file +}