From d148cd9215ad75c039ab9838ae42476e3a7c63a4 Mon Sep 17 00:00:00 2001 From: Siddhant Date: Wed, 7 Feb 2024 02:21:44 -0600 Subject: [PATCH 1/3] fix(r2): update Sippy API request payloads --- .changeset/five-cooks-share.md | 5 ++ packages/wrangler/src/__tests__/r2.test.ts | 37 +++++++++----- packages/wrangler/src/r2/helpers.ts | 57 ++++++++++++++-------- packages/wrangler/src/r2/sippy.ts | 44 ++++++++++------- 4 files changed, 92 insertions(+), 51 deletions(-) create mode 100644 .changeset/five-cooks-share.md diff --git a/.changeset/five-cooks-share.md b/.changeset/five-cooks-share.md new file mode 100644 index 0000000000..412081cda8 --- /dev/null +++ b/.changeset/five-cooks-share.md @@ -0,0 +1,5 @@ +--- +"wrangler": patch +--- + +Update API calls for Sippy's endpoints diff --git a/packages/wrangler/src/__tests__/r2.test.ts b/packages/wrangler/src/__tests__/r2.test.ts index f562da7caa..3abc84542b 100644 --- a/packages/wrangler/src/__tests__/r2.test.ts +++ b/packages/wrangler/src/__tests__/r2.test.ts @@ -326,12 +326,18 @@ describe("r2", () => { "*/accounts/some-account-id/r2/buckets/testBucket/sippy", async (request, response, context) => { expect(await request.json()).toEqual({ - access_key: "aws-secret", - bucket: "awsBucket", - key_id: "aws-key", - provider: "AWS", - r2_access_key: "some-secret", - r2_key_id: "some-key", + source: { + provider: "aws", + region: "awsRegion", + bucket: "awsBucket", + accessKeyId: "aws-key", + secretAccessKey: "aws-secret", + }, + destination: { + provider: "r2", + accessKeyId: "some-key", + secretAccessKey: "some-secret", + }, }); return response.once(context.json(createFetchResult({}))); } @@ -353,12 +359,17 @@ describe("r2", () => { "*/accounts/some-account-id/r2/buckets/testBucket/sippy", async (request, response, context) => { expect(await request.json()).toEqual({ - private_key: "gcs-private-key", - bucket: "gcsBucket", - client_email: "gcs-client-email", - provider: "GCS", - r2_access_key: "some-secret", - r2_key_id: "some-key", + source: { + provider: "gcs", + bucket: "gcsBucket", + clientEmail: "gcs-client-email", + privateKey: "gcs-private-key", + }, + destination: { + provider: "r2", + accessKeyId: "some-key", + secretAccessKey: "some-secret", + }, }); return response.once(context.json(createFetchResult({}))); } @@ -522,7 +533,7 @@ describe("r2", () => { ); await runWrangler("r2 bucket sippy get testBucket"); expect(std.out).toMatchInlineSnapshot( - `"Sippy upstream bucket: https://storage.googleapis.com/storage/v1/b/testBucket."` + `"Sippy configuration: https://storage.googleapis.com/storage/v1/b/testBucket"` ); }); }); diff --git a/packages/wrangler/src/r2/helpers.ts b/packages/wrangler/src/r2/helpers.ts index 5acee6e6ba..17da7448cb 100644 --- a/packages/wrangler/src/r2/helpers.ts +++ b/packages/wrangler/src/r2/helpers.ts @@ -224,6 +224,18 @@ export async function usingLocalBucket( } } +type SippyConfig = { + source: + | { provider: "aws"; region: string; bucket: string } + | { provider: "gcs"; bucket: string }; + destination: { + provider: "r2"; + account: string; + bucket: string; + accessKeyId: string; + }; +}; + /** * Retreive the sippy upstream bucket for the bucket with the given name */ @@ -231,7 +243,7 @@ export async function getR2Sippy( accountId: string, bucketName: string, jurisdiction?: string -): Promise { +): Promise { const headers: HeadersInit = {}; if (jurisdiction !== undefined) { headers["cf-r2-jurisdiction"] = jurisdiction; @@ -260,26 +272,27 @@ export async function deleteR2Sippy( ); } -export type R2Credentials = { - bucket: string; - r2_key_id: string; - r2_access_key: string; -}; - -export type SippyPutConfig = R2Credentials & - ( +export type SippyPutParams = { + source: | { - provider: "AWS"; - zone: string | undefined; - key_id: string; - access_key: string; + provider: "aws"; + region: string; + bucket: string; + accessKeyId: string; + secretAccessKey: string; } | { - provider: "GCS"; - client_email: string; - private_key: string; - } - ); + provider: "gcs"; + bucket: string; + clientEmail: string; + privateKey: string; + }; + destination: { + provider: "r2"; + accessKeyId: string; + secretAccessKey: string; + }; +}; /** * Enable sippy on the bucket with the given name @@ -287,15 +300,17 @@ export type SippyPutConfig = R2Credentials & export async function putR2Sippy( accountId: string, bucketName: string, - config: SippyPutConfig, + params: SippyPutParams, jurisdiction?: string ): Promise { - const headers: HeadersInit = {}; + const headers: HeadersInit = { + "Content-Type": "application/json", + }; if (jurisdiction !== undefined) { headers["cf-r2-jurisdiction"] = jurisdiction; } return await fetchResult( `/accounts/${accountId}/r2/buckets/${bucketName}/sippy`, - { method: "PUT", body: JSON.stringify(config), headers } + { method: "PUT", body: JSON.stringify(params), headers } ); } diff --git a/packages/wrangler/src/r2/sippy.ts b/packages/wrangler/src/r2/sippy.ts index 1a04d7a83d..2b4c7aef90 100644 --- a/packages/wrangler/src/r2/sippy.ts +++ b/packages/wrangler/src/r2/sippy.ts @@ -9,7 +9,7 @@ import type { CommonYargsArgv, StrictYargsOptionsToInterface, } from "../yargs-types"; -import type { SippyPutConfig } from "./helpers"; +import type { SippyPutParams } from "./helpers"; const NO_SUCH_OBJECT_KEY = 10007; const SIPPY_PROVIDER_CHOICES = ["AWS", "GCS"]; @@ -141,11 +141,7 @@ export async function EnableHandler( } } - let sippyConfig = { - bucket: args.bucket ?? "", - r2_key_id: args.r2KeyId ?? "", - r2_access_key: args.r2SecretAccessKey ?? "", - } as SippyPutConfig; + let sippyConfig: SippyPutParams; if (args.provider == "AWS") { if (!(args.keyId && args.secretAccessKey)) { @@ -154,11 +150,18 @@ export async function EnableHandler( ); } sippyConfig = { - ...sippyConfig, - provider: "AWS", - zone: args.region, - key_id: args.keyId, - access_key: args.secretAccessKey, + source: { + provider: "aws", + region: args.region, + bucket: args.bucket, + accessKeyId: args.accessKeyId, + secretAccessKey: args.secretAccessKey, + }, + destination: { + provider: "r2", + accessKeyId: args.r2AccessKeyId, + secretAccessKey: args.r2SecretAccessKey, + }, }; } else if (args.provider == "GCS") { if (args.serviceAccountKeyFile) { @@ -177,10 +180,17 @@ export async function EnableHandler( } args.privateKey = args.privateKey.replace(/\\n/g, "\n"); sippyConfig = { - ...sippyConfig, - provider: "GCS", - client_email: args.clientEmail, - private_key: args.privateKey, + source: { + provider: "gcs", + bucket: args.bucket, + clientEmail: args.clientEmail, + privateKey: args.privateKey, + }, + destination: { + provider: "r2", + accessKeyId: args.r2AccessKeyId, + secretAccessKey: args.r2SecretAccessKey, + }, }; } @@ -211,12 +221,12 @@ export async function GetHandler( const accountId = await requireAuth(config); try { - const sippyBucket = await getR2Sippy( + const sippyConfig = await getR2Sippy( accountId, args.name, args.jurisdiction ); - logger.log(`Sippy upstream bucket: ${sippyBucket}.`); + logger.log("Sippy configuration:", sippyConfig); } catch (e) { if (e instanceof APIError && "code" in e && e.code == NO_SUCH_OBJECT_KEY) { logger.log(`No Sippy configuration found for the '${args.name}' bucket.`); From 50b52eba147e6ac79b2cb304d38527fd159ea520 Mon Sep 17 00:00:00 2001 From: Siddhant Date: Wed, 7 Feb 2024 02:22:59 -0600 Subject: [PATCH 2/3] improve(r2): rename sippy flags for clarity --- packages/wrangler/src/__tests__/r2.test.ts | 8 ++++---- packages/wrangler/src/r2/sippy.ts | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/wrangler/src/__tests__/r2.test.ts b/packages/wrangler/src/__tests__/r2.test.ts index 3abc84542b..6190695c27 100644 --- a/packages/wrangler/src/__tests__/r2.test.ts +++ b/packages/wrangler/src/__tests__/r2.test.ts @@ -344,7 +344,7 @@ describe("r2", () => { ) ); await runWrangler( - "r2 bucket sippy enable testBucket --r2-key-id=some-key --r2-secret-access-key=some-secret --provider=AWS --key-id=aws-key --secret-access-key=aws-secret --bucket=awsBucket" + "r2 bucket sippy enable testBucket --r2-access-key-id=some-key --r2-secret-access-key=some-secret --provider=AWS --access-key-id=aws-key --secret-access-key=aws-secret --region=awsRegion --bucket=awsBucket" ); expect(std.out).toMatchInlineSnapshot( `"✨ Successfully enabled Sippy on the 'testBucket' bucket."` @@ -376,7 +376,7 @@ describe("r2", () => { ) ); await runWrangler( - "r2 bucket sippy enable testBucket --r2-key-id=some-key --r2-secret-access-key=some-secret --provider=GCS --client-email=gcs-client-email --private-key=gcs-private-key --bucket=gcsBucket" + "r2 bucket sippy enable testBucket --r2-access-key-id=some-key --r2-secret-access-key=some-secret --provider=GCS --client-email=gcs-client-email --private-key=gcs-private-key --bucket=gcsBucket" ); expect(std.out).toMatchInlineSnapshot( `"✨ Successfully enabled Sippy on the 'testBucket' bucket."` @@ -410,12 +410,12 @@ describe("r2", () => { --provider [choices: \\"AWS\\", \\"GCS\\"] --bucket The name of the upstream bucket [string] --region (AWS provider only) The region of the upstream bucket [string] - --key-id (AWS provider only) The secret access key id for the upstream bucket [string] + --access-key-id (AWS provider only) The secret access key id for the upstream bucket [string] --secret-access-key (AWS provider only) The secret access key for the upstream bucket [string] --service-account-key-file (GCS provider only) The path to your Google Cloud service account key JSON file [string] --client-email (GCS provider only) The client email for your Google Cloud service account key [string] --private-key (GCS provider only) The private key for your Google Cloud service account key [string] - --r2-key-id The secret access key id for this R2 bucket [string] + --r2-access-key-id The secret access key id for this R2 bucket [string] --r2-secret-access-key The secret access key for this R2 bucket [string]" `); expect(std.err).toMatchInlineSnapshot(` diff --git a/packages/wrangler/src/r2/sippy.ts b/packages/wrangler/src/r2/sippy.ts index 2b4c7aef90..dfc9f615e6 100644 --- a/packages/wrangler/src/r2/sippy.ts +++ b/packages/wrangler/src/r2/sippy.ts @@ -38,7 +38,7 @@ export function EnableOptions(yargs: CommonYargsArgv) { description: "(AWS provider only) The region of the upstream bucket", string: true, }) - .option("key-id", { + .option("access-key-id", { description: "(AWS provider only) The secret access key id for the upstream bucket", string: true, @@ -63,7 +63,7 @@ export function EnableOptions(yargs: CommonYargsArgv) { "(GCS provider only) The private key for your Google Cloud service account key", string: true, }) - .option("r2-key-id", { + .option("r2-access-key-id", { description: "The secret access key id for this R2 bucket", string: true, }) @@ -101,10 +101,10 @@ export async function EnableHandler( args.region ??= await prompt( "Enter the AWS region where your S3 bucket is located (example: us-west-2):" ); - args.keyId ??= await prompt( + args.accessKeyId ??= await prompt( "Enter your AWS Access Key ID (requires read and list access):" ); - if (!args.keyId) { + if (!args.accessKeyId) { throw new UserError("Must specify an AWS Access Key ID."); } args.secretAccessKey ??= await prompt( @@ -129,10 +129,10 @@ export async function EnableHandler( } } - args.r2KeyId ??= await prompt( + args.r2AccessKeyId ??= await prompt( "Enter your R2 Access Key ID (requires read and write access):" ); - if (!args.r2KeyId) { + if (!args.r2AccessKeyId) { throw new UserError("Must specify an R2 Access Key ID."); } args.r2SecretAccessKey ??= await prompt("Enter your R2 Secret Access Key:"); From 5cb034b152b281afc6d8306ee307db706039ea84 Mon Sep 17 00:00:00 2001 From: Siddhant Date: Wed, 7 Feb 2024 11:11:51 -0600 Subject: [PATCH 3/3] improve(r2): stricter existence in Sippy --- packages/wrangler/src/r2/sippy.ts | 54 ++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/packages/wrangler/src/r2/sippy.ts b/packages/wrangler/src/r2/sippy.ts index dfc9f615e6..233cf5dbe1 100644 --- a/packages/wrangler/src/r2/sippy.ts +++ b/packages/wrangler/src/r2/sippy.ts @@ -97,10 +97,13 @@ export async function EnableHandler( throw new UserError(`Must specify ${args.provider} bucket name.`); } - if (args.provider == "AWS") { + if (args.provider === "AWS") { args.region ??= await prompt( "Enter the AWS region where your S3 bucket is located (example: us-west-2):" ); + if (!args.region) { + throw new UserError("Must specify an AWS Region."); + } args.accessKeyId ??= await prompt( "Enter your AWS Access Key ID (requires read and list access):" ); @@ -113,7 +116,7 @@ export async function EnableHandler( if (!args.secretAccessKey) { throw new UserError("Must specify an AWS Secret Access Key."); } - } else if (args.provider == "GCS") { + } else if (args.provider === "GCS") { if ( !(args.clientEmail && args.privateKey) && !args.serviceAccountKeyFile @@ -143,12 +146,18 @@ export async function EnableHandler( let sippyConfig: SippyPutParams; - if (args.provider == "AWS") { - if (!(args.keyId && args.secretAccessKey)) { - throw new UserError( - `Error: must provide --key-id and --secret-access-key.` - ); - } + if (args.provider === "AWS") { + if (!args.region) throw new UserError("Error: must provide --region."); + if (!args.bucket) throw new UserError("Error: must provide --bucket."); + if (!args.accessKeyId) + throw new UserError("Error: must provide --access-key-id."); + if (!args.secretAccessKey) + throw new UserError("Error: must provide --secret-access-key."); + if (!args.r2AccessKeyId) + throw new UserError("Error: must provide --r2-access-key-id."); + if (!args.r2SecretAccessKey) + throw new UserError("Error: must provide --r2-secret-access-key."); + sippyConfig = { source: { provider: "aws", @@ -163,22 +172,33 @@ export async function EnableHandler( secretAccessKey: args.r2SecretAccessKey, }, }; - } else if (args.provider == "GCS") { + } else if (args.provider === "GCS") { if (args.serviceAccountKeyFile) { const serviceAccount = JSON.parse( readFileSync(args.serviceAccountKeyFile) ); if ("client_email" in serviceAccount && "private_key" in serviceAccount) { - args.clientEmail = serviceAccount["client_email"]; - args.privateKey = serviceAccount["private_key"]; + args.clientEmail = serviceAccount.client_email; + args.privateKey = serviceAccount.private_key; } } - if (!(args.clientEmail && args.privateKey)) { + + if (!args.bucket) throw new UserError("Error: must provide --bucket."); + if (!args.clientEmail) throw new UserError( - `Error: must provide --service-account-key-file or --client-email and --private-key.` + "Error: must provide --service-account-key-file or --client-email." + ); + if (!args.privateKey) + throw new UserError( + "Error: must provide --service-account-key-file or --private-key." ); - } args.privateKey = args.privateKey.replace(/\\n/g, "\n"); + + if (!args.r2AccessKeyId) + throw new UserError("Error: must provide --r2-access-key-id."); + if (!args.r2SecretAccessKey) + throw new UserError("Error: must provide --r2-secret-access-key."); + sippyConfig = { source: { provider: "gcs", @@ -192,6 +212,10 @@ export async function EnableHandler( secretAccessKey: args.r2SecretAccessKey, }, }; + } else { + throw new UserError( + "Error: unrecognized provider. Possible options are AWS & GCS." + ); } await putR2Sippy(accountId, args.name, sippyConfig, args.jurisdiction); @@ -228,7 +252,7 @@ export async function GetHandler( ); logger.log("Sippy configuration:", sippyConfig); } catch (e) { - if (e instanceof APIError && "code" in e && e.code == NO_SUCH_OBJECT_KEY) { + if (e instanceof APIError && "code" in e && e.code === NO_SUCH_OBJECT_KEY) { logger.log(`No Sippy configuration found for the '${args.name}' bucket.`); } else { throw e;