From 6a4facecb8f7ae4a4cde53e20acedba3f81240f4 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:35:53 +0200 Subject: [PATCH 01/20] feat(controlplane): enable using aws based s3 storage --- controlplane/src/core/build-server.ts | 26 ++--- controlplane/src/core/env.schema.ts | 20 +++- controlplane/src/core/util.ts | 47 ++++++++ controlplane/src/index.ts | 8 +- controlplane/test/authentication.test.ts | 6 +- controlplane/test/utils.s3storage.test.ts | 134 ++++++++++++++++++++++ 6 files changed, 223 insertions(+), 18 deletions(-) create mode 100644 controlplane/test/utils.s3storage.test.ts diff --git a/controlplane/src/core/build-server.ts b/controlplane/src/core/build-server.ts index 25b9ee7f79..e5366d1c55 100644 --- a/controlplane/src/core/build-server.ts +++ b/controlplane/src/core/build-server.ts @@ -37,7 +37,7 @@ import { BillingRepository } from './repositories/BillingRepository.js'; import { BillingService } from './services/BillingService.js'; import { UserRepository } from './repositories/UserRepository.js'; import { AIGraphReadmeQueue, createAIGraphReadmeWorker } from './workers/AIGraphReadmeWorker.js'; -import { fastifyLoggerId } from './util.js'; +import { fastifyLoggerId, createS3ClientConfig, extractS3BucketName } from './util.js'; import { ApiKeyRepository } from './repositories/ApiKeyRepository.js'; import { createDeleteOrganizationWorker, DeleteOrganizationQueue } from './workers/DeleteOrganizationWorker.js'; @@ -86,7 +86,11 @@ export interface BuildConfig { }; slack: { clientID?: string; clientSecret?: string }; cdnBaseUrl: string; - s3StorageUrl: string; + s3Storage: { + url: string; + endpoint?: string; + region?: string; + }; mailer: { smtpEnabled: boolean; smtpHost?: string; @@ -343,22 +347,14 @@ export default async function build(opts: BuildConfig) { }); } - if (!opts.s3StorageUrl) { + if (!opts.s3Storage || !opts.s3Storage.url) { throw new Error('S3 storage URL is required'); } - const url = new URL(opts.s3StorageUrl); - const s3Client = new S3Client({ - // For AWS S3, the region can be set via the endpoint - region: 'auto', - endpoint: url.origin, - credentials: { - accessKeyId: url.username ?? '', - secretAccessKey: url.password ?? '', - }, - forcePathStyle: true, - }); - const bucketName = url.pathname.slice(1); + const bucketName = extractS3BucketName(opts.s3Storage.url); + const s3Config = createS3ClientConfig(opts.s3Storage.url, bucketName, opts.s3Storage.region, opts.s3Storage.endpoint); + + const s3Client = new S3Client(s3Config); const blobStorage = new S3BlobStorage(s3Client, bucketName); /** diff --git a/controlplane/src/core/env.schema.ts b/controlplane/src/core/env.schema.ts index df75beae5b..a65a3fa4c5 100644 --- a/controlplane/src/core/env.schema.ts +++ b/controlplane/src/core/env.schema.ts @@ -99,9 +99,27 @@ export const envVariables = z SLACK_APP_CLIENT_ID: z.string().optional(), SLACK_APP_CLIENT_SECRET: z.string().optional(), /** - * S3 Storage e.g. for persistent operations + * S3 Storage e.g. for persistent operations. + * + * S3_STORAGE_URL: The blobStorage url containing username and password (e.g.: https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com) + * S3_REGION: The region to use for the S3 storage (e.g.: us-east-1, this fallbacks to auto and must be set when using aws) + * S3_ENDPOINT: The aws endpoint to use for the S3 storage (e.g.: s3.amazonaws.com, this fallbacks to the origin of the S3_STORAGE_URL) + * + * Examples: + * Minio Storage + * S3_STORAGE_URL="http://minio:pass@minio:9000/cosmo" + * S3_REGION="auto" # default + * S3_ENDPOINT=S3_STORAGE_URL.origin # default + * + * AWS S3 Storage + * S3_STORAGE_URL="https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com" + * S3_REGION="us-east-1" # set this for amazon to your region + * S3_ENDPOINT="s3.amazonaws.com" # replaces the bucket from the S3_STORAGE_URL origin */ S3_STORAGE_URL: z.string(), + S3_ENDPOINT: z.string().optional(), + S3_REGION: z.string().default('auto'), + /** * Email */ diff --git a/controlplane/src/core/util.ts b/controlplane/src/core/util.ts index c2390557f2..9d5a84ca62 100644 --- a/controlplane/src/core/util.ts +++ b/controlplane/src/core/util.ts @@ -1,4 +1,5 @@ import { randomFill } from 'node:crypto'; +import { S3ClientConfig } from '@aws-sdk/client-s3'; import { HandlerContext } from '@connectrpc/connect'; import { GraphQLSubscriptionProtocol, @@ -371,3 +372,49 @@ export function getValueOrDefault(map: Map, key: K, constructor: () export function webhookAxiosRetryCond(err: AxiosError) { return isNetworkError(err) || isRetryableError(err); } + +export function createS3ClientConfig( + s3Url: string, + bucketName: string, + region: string | undefined, + endpoint: string | undefined, +): S3ClientConfig { + const url = new URL(s3Url); + const forcePathStyle = !isVirtualHostStyleUrl(url); + + const accessKeyId = url.username ?? ''; + const secretAccessKey = url.password ?? ''; + + if (forcePathStyle && !endpoint) { + endpoint = url.origin; + } + + if (!forcePathStyle && !endpoint) { + endpoint = url.origin.replace(`${bucketName}.`, ''); + } + + return { + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle, + }; +} + +export function extractS3BucketName(s3Url: string) { + const url = new URL(s3Url); + + if (isVirtualHostStyleUrl(url)) { + return url.hostname.split('.')[0]; + } + + // path based style + return url.pathname.slice(1); +} + +export function isVirtualHostStyleUrl(url: URL) { + return url.hostname.split('.').length > 2; +} diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index 36ab77a41d..b05a9c9181 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -41,6 +41,8 @@ const { SLACK_APP_CLIENT_ID, SLACK_APP_CLIENT_SECRET, S3_STORAGE_URL, + S3_ENDPOINT, + S3_REGION, SMTP_ENABLED, SMTP_HOST, SMTP_PORT, @@ -116,7 +118,11 @@ const options: BuildConfig = { clientID: SLACK_APP_CLIENT_ID, clientSecret: SLACK_APP_CLIENT_SECRET, }, - s3StorageUrl: S3_STORAGE_URL, + s3Storage: { + url: S3_STORAGE_URL, + region: S3_REGION, + endpoint: S3_ENDPOINT || '', + }, mailer: { smtpEnabled: SMTP_ENABLED, smtpHost: SMTP_HOST, diff --git a/controlplane/test/authentication.test.ts b/controlplane/test/authentication.test.ts index a8e997ce1a..6ee21afa6d 100644 --- a/controlplane/test/authentication.test.ts +++ b/controlplane/test/authentication.test.ts @@ -48,7 +48,11 @@ describe('Authentication', (ctx) => { clientID: '', clientSecret: '', }, - s3StorageUrl: 'http://localhost:9000', + s3Storage: { + url: 'http://localhost:9000', + region: 'auto', + endpoint: 'localhost:9000', + }, mailer: { smtpEnabled: false, smtpHost: '', diff --git a/controlplane/test/utils.s3storage.test.ts b/controlplane/test/utils.s3storage.test.ts new file mode 100644 index 0000000000..4c928d8810 --- /dev/null +++ b/controlplane/test/utils.s3storage.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, test } from 'vitest'; +import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src/core/util.js'; + +describe('S3 Utils', () => { + describe('createS3ClientConfig', () => { + test('that it correctly configures an S3 client for a path-style URL', () => { + const s3Url = 'http://username:password@minio:9000/cosmo'; + const bucketName = 'cosmo'; + const region = 'auto'; + const endpoint = undefined; + + const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + + expect(config).toEqual({ + region: 'auto', + endpoint: 'http://minio:9000', + credentials: { + accessKeyId: 'username', + secretAccessKey: 'password', + }, + forcePathStyle: true, + }); + }); + + test('that it correctly configures an S3 client for a virtual-hosted-style URL with provided endpoint', () => { + const s3Url = 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com'; + const bucketName = 'cosmo-controlplane-bucket'; + const region = 'us-east-1'; + const endpoint = 's3.amazonaws.com'; + + const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + + expect(config).toEqual({ + region: 'us-east-1', + endpoint: 's3.amazonaws.com', + credentials: { + accessKeyId: 'username', + secretAccessKey: 'password', + }, + forcePathStyle: false, + }); + }); + + test('that it correctly configures an S3 client for a virtual-hosted-style URL without provided endpoint', () => { + const s3Url = 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com'; + const bucketName = 'cosmo-controlplane-bucket'; + const region = 'us-east-1'; + const endpoint = undefined; + + const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + + expect(config).toEqual({ + region: 'us-east-1', + endpoint: 'https://s3.amazonaws.com', + credentials: { + accessKeyId: 'username', + secretAccessKey: 'password', + }, + forcePathStyle: false, + }); + }); + + test('that it handles missing username and password in the URL correctly', () => { + const s3Url = 'https://cosmo-controlplane-bucket.s3.amazonaws.com'; + const bucketName = 'cosmo-controlplane-bucket'; + const region = 'us-east-1'; + const endpoint = ''; + + const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + + expect(config).toEqual({ + region: 'us-east-1', + endpoint: 'https://s3.amazonaws.com', + credentials: { + accessKeyId: '', + secretAccessKey: '', + }, + forcePathStyle: false, + }); + }); + }); + + describe('extractS3BucketName', () => { + test('that it returns the correct bucket name for a virtual-hosted-style URL', () => { + const s3Url = 'https://cosmo-controlplane-bucket.s3.amazonaws.com/some/object'; + + const bucketName = extractS3BucketName(s3Url); + + expect(bucketName).toBe('cosmo-controlplane-bucket'); + }); + + test('that it returns the correct bucket name for a path-style URL', () => { + const s3Url = 'http://minio:9000/cosmo'; + + const bucketName = extractS3BucketName(s3Url); + + expect(bucketName).toBe('cosmo'); + }); + + test('that it returns the correct bucket name when the URL has multiple path segments', () => { + const s3Url = 'http://username:password@localhost:9000/foo'; + + const bucketName = extractS3BucketName(s3Url); + + expect(bucketName).toBe('foo'); + }); + }); + + describe('isVirtualHostStyleUrl', () => { + test('that it returns true for a virtual-hosted-style URL', () => { + const url = new URL('https://cosmo-controlplane-bucket.s3.amazonaws.com'); + + const result = isVirtualHostStyleUrl(url); + + expect(result).toBe(true); + }); + + test('that it returns false for a path-style URL', () => { + const url = new URL('http://minio:9000/cosmo'); + + const result = isVirtualHostStyleUrl(url); + + expect(result).toBe(false); + }); + + test('that it returns false for a custom domain without bucket name in the hostname', () => { + const url = new URL('https://example.com/cosmo'); + + const result = isVirtualHostStyleUrl(url); + + expect(result).toBe(false); + }); + }); +}); From ac0e507577ad027533b36e4ffce1d3c9b8c07ef7 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:37:00 +0200 Subject: [PATCH 02/20] feat(cdn-server): enable aws s3 storage backend --- cdn-server/.env.example | 5 ++++- cdn-server/src/s3.ts | 19 +++++++--------- cdn-server/src/utils.ts | 48 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 cdn-server/src/utils.ts diff --git a/cdn-server/.env.example b/cdn-server/.env.example index 2ec2a8a5e8..49b473f69a 100644 --- a/cdn-server/.env.example +++ b/cdn-server/.env.example @@ -1,4 +1,7 @@ PORT=11000 AUTH_JWT_SECRET=fkczyomvdprgvtmvkuhvprxuggkbgwld AUTH_ADMISSION_JWT_SECRET="uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW" -S3_STORAGE_URL=http://minio:changeme@localhost:10000/cosmo \ No newline at end of file + +S3_STORAGE_URL=http://minio:changeme@localhost:10000/cosmo +S3_REGION='auto' +S3_ENDPOINT="http://localhost:10000" \ No newline at end of file diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index bd6a7c7fac..78684fba7c 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -1,6 +1,7 @@ import { GetObjectCommand, HeadObjectCommand, NoSuchKey, NotFound, S3Client } from '@aws-sdk/client-s3'; import { BlobNotFoundError, BlobObject, BlobStorage } from '@wundergraph/cosmo-cdn'; import { Context } from 'hono'; +import { createS3ClientConfig, extractS3BucketName } from './utils'; /** * Retrieves objects from S3 given an S3Client and a bucket name @@ -86,16 +87,12 @@ class S3BlobStorage implements BlobStorage { export const createS3BlobStorage = (storageUrl: string): BlobStorage => { const url = new URL(storageUrl); - const region = url.searchParams.get('region') ?? 'default'; - const s3Client = new S3Client({ - region, - endpoint: url.origin, - credentials: { - accessKeyId: url.username ?? '', - secretAccessKey: url.password ?? '', - }, - forcePathStyle: true, - }); - const bucketName = url.pathname.slice(1); + const region = url.searchParams.get('region') ?? process.env.S3_REGION ?? 'default'; + const endpoint = url.searchParams.get('endpoint') ?? process.env.S3_ENDPOINT; + + const bucketName = extractS3BucketName(storageUrl); + const s3Config = createS3ClientConfig(storageUrl, bucketName, region, endpoint); + const s3Client = new S3Client(s3Config); + return new S3BlobStorage(s3Client, bucketName); }; diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts new file mode 100644 index 0000000000..cd454af89d --- /dev/null +++ b/cdn-server/src/utils.ts @@ -0,0 +1,48 @@ +import { S3ClientConfig } from '@aws-sdk/client-s3'; + +// see: controlplane/test/utils.s3storage.test.ts +export function createS3ClientConfig( + s3Url: string, + bucketName: string, + region: string | undefined, + endpoint: string | undefined, +): S3ClientConfig { + const url = new URL(s3Url); + const forcePathStyle = !isVirtualHostStyleUrl(url); + + const accessKeyId = url.username ?? ''; + const secretAccessKey = url.password ?? ''; + + if (forcePathStyle && !endpoint) { + endpoint = url.origin; + } + + if (!forcePathStyle && !endpoint) { + endpoint = url.origin.replace(`${bucketName}.`, ''); + } + + return { + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle, + }; +} + +export function extractS3BucketName(s3Url: string) { + const url = new URL(s3Url); + + if (isVirtualHostStyleUrl(url)) { + return url.hostname.split('.')[0]; + } + + // path based style + return url.pathname.slice(1); +} + +export function isVirtualHostStyleUrl(url: URL) { + return url.hostname.split('.').length > 2; +} From ebb29b5a44935c5f1fa3235e3fd15e0102a1cb55 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:44:41 +0200 Subject: [PATCH 03/20] feat(cdn): enable s3 region and endpoint handling --- helm/cosmo/charts/cdn/README.md | 3 ++- helm/cosmo/charts/cdn/templates/config-map.yaml | 4 ++++ helm/cosmo/charts/cdn/templates/deployment.yaml | 14 ++++++++++++++ helm/cosmo/charts/cdn/values.yaml | 4 ++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/helm/cosmo/charts/cdn/README.md b/helm/cosmo/charts/cdn/README.md index 60572a5728..71468cc6e5 100644 --- a/helm/cosmo/charts/cdn/README.md +++ b/helm/cosmo/charts/cdn/README.md @@ -16,7 +16,8 @@ WunderGraph Cosmo CDN | autoscaling.minReplicas | int | `1` | | | autoscaling.targetCPUUtilizationPercentage | int | `80` | | | commonLabels | object | `{}` | Add labels to all deployed resources | -| configuration | string | `nil` | | +| configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | deploymentStrategy | object | `{}` | | | existingSecret | string | `""` | Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. | | extraEnvVars | list | `[]` | Allows to set additional environment variables on the container. Useful for global application non-specific settings. | diff --git a/helm/cosmo/charts/cdn/templates/config-map.yaml b/helm/cosmo/charts/cdn/templates/config-map.yaml index 29c8578664..ce0c806ab6 100644 --- a/helm/cosmo/charts/cdn/templates/config-map.yaml +++ b/helm/cosmo/charts/cdn/templates/config-map.yaml @@ -10,3 +10,7 @@ metadata: {{- include "cdn.labels" . | nindent 4 }} data: port: "{{ .Values.service.port }}" + s3Region: "{{ .Values.configuration.s3Region }}" + {{- if .Values.configuration.s3Endpoint }} + s3Endpoint: "{{ .Values.configuration.s3Endpoint }}" + {{- end }} diff --git a/helm/cosmo/charts/cdn/templates/deployment.yaml b/helm/cosmo/charts/cdn/templates/deployment.yaml index 191721d5db..940c65828c 100644 --- a/helm/cosmo/charts/cdn/templates/deployment.yaml +++ b/helm/cosmo/charts/cdn/templates/deployment.yaml @@ -59,6 +59,20 @@ spec: name: {{ include "cdn.fullname" . }}-configmap key: port + - name: S3_REGION + valueFrom: + configMapKeyRef: + name: {{ include "cdn.fullname" . }}-configmap + key: s3Region + + {{- if .Values.configuration.s3Endpoint }} + - name: S3_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ include "cdn.fullname" . }}-configmap + key: s3Endpoint + {{- end }} + - name: S3_STORAGE_URL valueFrom: secretKeyRef: diff --git a/helm/cosmo/charts/cdn/values.yaml b/helm/cosmo/charts/cdn/values.yaml index e0e189b7fa..80556ff256 100644 --- a/helm/cosmo/charts/cdn/values.yaml +++ b/helm/cosmo/charts/cdn/values.yaml @@ -144,6 +144,10 @@ probes: ############################# configuration: + # -- The region where the S3 bucket is located. + s3Region: 'auto' + # -- The endpoint of the S3 bucket. + s3Endpoint: '' # -- Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. existingSecret: "" From 6a5c6f6957100cc188c5b5acfcbea4370e3cdeea Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:46:25 +0200 Subject: [PATCH 04/20] feat(controlplane): enable s3 region and endpoint handling --- .../charts/controlplane/templates/config-map.yaml | 6 +++++- .../charts/controlplane/templates/deployment.yaml | 12 ++++++++++++ helm/cosmo/charts/controlplane/values.yaml | 4 ++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/helm/cosmo/charts/controlplane/templates/config-map.yaml b/helm/cosmo/charts/controlplane/templates/config-map.yaml index 6a2244a1a7..2fe430080e 100644 --- a/helm/cosmo/charts/controlplane/templates/config-map.yaml +++ b/helm/cosmo/charts/controlplane/templates/config-map.yaml @@ -40,4 +40,8 @@ data: prometheusHost: "{{ .Values.configuration.prometheus.host }}" prometheusPort: "{{ .Values.configuration.prometheus.port }}" prometheusPath: "{{ .Values.configuration.prometheus.path }}" - {{ end }} + {{- end }} + s3Region: "{{ .Values.configuration.s3Region }}" + {{- if .Values.configuration.s3Endpoint }} + s3Endpoint: "{{ .Values.configuration.s3Endpoint }}" + {{ end }} \ No newline at end of file diff --git a/helm/cosmo/charts/controlplane/templates/deployment.yaml b/helm/cosmo/charts/controlplane/templates/deployment.yaml index 3877c0e1d2..313a5631e7 100644 --- a/helm/cosmo/charts/controlplane/templates/deployment.yaml +++ b/helm/cosmo/charts/controlplane/templates/deployment.yaml @@ -226,6 +226,18 @@ spec: name: {{ include "controlplane.secretName" . }} key: s3StorageUrl {{- end }} + - name: S3_REGION + valueFrom: + configMapKeyRef: + name: {{ include "controlplane.fullname" . }}-configmap + key: s3Region + {{- if .Values.configuration.s3Endpoint }} + - name: S3_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ include "controlplane.fullname" . }}-configmap + key: s3Endpoint + {{- end }} - name: SMTP_ENABLED valueFrom: configMapKeyRef: diff --git a/helm/cosmo/charts/controlplane/values.yaml b/helm/cosmo/charts/controlplane/values.yaml index 2e556f3f98..fe23642156 100644 --- a/helm/cosmo/charts/controlplane/values.yaml +++ b/helm/cosmo/charts/controlplane/values.yaml @@ -184,6 +184,10 @@ configuration: slackAppClientId: '' slackAppClientSecret: '' s3StorageUrl: 'http://minio:changeme@cosmo-minio:9000/cosmo' + # -- The region where the S3 bucket is located. + s3Region: 'auto' + # -- The endpoint of the S3 bucket. + s3Endpoint: '' stripeSecretKey: '' stripeWebhookSecret: '' # -- The default billing plan, eg `developer@1` From 85daa7bebf39029b5b6121b275fbc5698ef7670c Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:47:19 +0200 Subject: [PATCH 05/20] feat: enable s3region and s3Endpoint for cdn and controlplane --- helm/cosmo/values.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/helm/cosmo/values.yaml b/helm/cosmo/values.yaml index 6cd826c7d3..5bb3c4b1a6 100644 --- a/helm/cosmo/values.yaml +++ b/helm/cosmo/values.yaml @@ -120,6 +120,10 @@ cdn: configuration: # This must match the value used in controlplane.configuration.s3StorageUrl s3StorageUrl: 'http://minio:changeme@cosmo-minio:9000/cosmo' + # -- The region where the S3 bucket is located. + s3Region: 'auto' + # -- The endpoint of the S3 bucket. + s3Endpoint: '' # Cosmo Controlplane. For more options, please refer to the README.md controlplane: @@ -184,7 +188,13 @@ controlplane: clickhouseMigrationDsn: 'clickhouse://default:changeme@cosmo-clickhouse:9000/cosmo?dial_timeout=15s&max_execution_time=60' redisHost: 'cosmo-redis-master' redisPort: 6379 + # This must match the value used in cdn.configuration.s3StorageUrl s3StorageUrl: 'http://minio:changeme@cosmo-minio:9000/cosmo' + # -- The region where the S3 bucket is located. + s3Region: 'auto' + # -- The endpoint of the S3 bucket. + s3Endpoint: '' + cdnBaseUrl: 'http://cosmo-cdn:8787' # -- Use this section to configure the smtp server. From 2e8cce483f7dcb77be283268a80920ef5e17fc8b Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:48:12 +0200 Subject: [PATCH 06/20] chore: update docs --- helm/cosmo/README.md | 4 ++++ helm/cosmo/charts/controlplane/README.md | 2 ++ 2 files changed, 6 insertions(+) diff --git a/helm/cosmo/README.md b/helm/cosmo/README.md index 410c9eb395..00a5d82280 100644 --- a/helm/cosmo/README.md +++ b/helm/cosmo/README.md @@ -35,6 +35,8 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | Key | Type | Default | Description | |-----|------|---------|-------------| | cdn.commonLabels | object | `{}` | Add labels to all deployed resources | +| cdn.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| cdn.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | cdn.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | | clickhouse.auth.password | string | `"changeme"` | | | clickhouse.auth.username | string | `"default"` | | @@ -64,6 +66,8 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | controlplane.configuration.prometheus.port | int | `8088` | The port where metrics are exposed. Default is port 8088. | | controlplane.configuration.redisHost | string | `"cosmo-redis-master"` | | | controlplane.configuration.redisPort | int | `6379` | | +| controlplane.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| controlplane.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | controlplane.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | | controlplane.configuration.smtp | object | `{"enabled":false,"host":"smtp.postmarkapp.com","password":"","port":587,"requireTls":true,"secure":true,"username":""}` | Use this section to configure the smtp server. | | controlplane.configuration.smtp.enabled | bool | `false` | Enables the smtp server. Default is false. | diff --git a/helm/cosmo/charts/controlplane/README.md b/helm/cosmo/charts/controlplane/README.md index 1aac745133..915deab74d 100644 --- a/helm/cosmo/charts/controlplane/README.md +++ b/helm/cosmo/charts/controlplane/README.md @@ -49,6 +49,8 @@ WunderGraph Cosmo Controlplane | configuration.redisTlsCa | string | `""` | When connecting to a redis instance over TLS. Accept a cert in PEM format (as one-line with \n) or file. | | configuration.redisTlsCert | string | `""` | | | configuration.redisTlsKey | string | `""` | | +| configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | | configuration.slackAppClientId | string | `""` | | | configuration.slackAppClientSecret | string | `""` | | From f8aab09dcd3ec22e6bfd818e381d1009311ec7d0 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Sun, 1 Sep 2024 22:51:02 +0200 Subject: [PATCH 07/20] chore: enable S3 handling in compose files --- docker-compose.full.yml | 13 ++++++++----- docker-compose.yml | 9 +++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docker-compose.full.yml b/docker-compose.full.yml index 6aa1f0af9a..aeb65930c0 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -205,10 +205,12 @@ services: context: . dockerfile: cdn-server/Dockerfile environment: - - PORT=11000 - - AUTH_JWT_SECRET=fkczyomvdprgvtmvkuhvprxuggkbgwld - - AUTH_ADMISSION_JWT_SECRET="uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW" - - S3_STORAGE_URL=http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo + PORT: 11000 + NODE_ENV: development + AUTH_JWT_SECRET: fkczyomvdprgvtmvkuhvprxuggkbgwld + AUTH_ADMISSION_JWT_SECRET: uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW + S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo} + S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} ports: - '11000:11000' networks: @@ -270,7 +272,8 @@ services: KC_API_URL: 'http://keycloak:8080' KC_FRONTEND_URL: 'http://localhost:8080' PROMETHEUS_API_URL: 'http://admin:test@prometheus:9090/api/v1' - S3_STORAGE_URL: http://minio:changeme@minio:9000/cosmo + S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo} + S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} CDN_BASE_URL: 'http://cdn:11000' REDIS_HOST: redis REDIS_PORT: 6379 diff --git a/docker-compose.yml b/docker-compose.yml index 0150805af0..1a6f809739 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,10 +164,11 @@ services: context: . dockerfile: cdn-server/Dockerfile environment: - - PORT=11000 - - AUTH_JWT_SECRET=fkczyomvdprgvtmvkuhvprxuggkbgwld - - AUTH_ADMISSION_JWT_SECRET=uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW - - S3_STORAGE_URL=http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo + PORT: 11000 + AUTH_JWT_SECRET: fkczyomvdprgvtmvkuhvprxuggkbgwld + AUTH_ADMISSION_JWT_SECRET: uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW + S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo} + S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} ports: - '11000:11000' networks: From 7bf5dbffade3f39c44d9e62f6c7be4218f8bc95d Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 08:25:01 +0200 Subject: [PATCH 08/20] chore: remove unecessary defaulting --- controlplane/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index b05a9c9181..fcefaefdf0 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -121,7 +121,7 @@ const options: BuildConfig = { s3Storage: { url: S3_STORAGE_URL, region: S3_REGION, - endpoint: S3_ENDPOINT || '', + endpoint: S3_ENDPOINT, }, mailer: { smtpEnabled: SMTP_ENABLED, From 9cac8fefadc336133083a966a897d88d0d9e72c2 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 10:37:34 +0200 Subject: [PATCH 09/20] feat: enable using aws credential handling for s3 --- cdn-server/.env.example | 5 +- cdn-server/src/s3.ts | 6 +- cdn-server/src/utils.ts | 32 +++--- controlplane/.env.example | 4 + controlplane/src/core/build-server.ts | 4 +- controlplane/src/core/env.schema.ts | 7 ++ controlplane/src/core/util.ts | 26 +++-- controlplane/src/types/index.ts | 8 ++ controlplane/test/utils.s3storage.test.ts | 100 +++++++++++++----- helm/cosmo/README.md | 4 + helm/cosmo/charts/cdn/README.md | 2 + .../charts/cdn/templates/deployment.yaml | 16 +++ helm/cosmo/charts/cdn/templates/secret.yaml | 6 ++ helm/cosmo/charts/cdn/values.yaml | 4 + helm/cosmo/charts/controlplane/README.md | 2 + .../controlplane/templates/deployment.yaml | 14 +++ .../charts/controlplane/templates/secret.yaml | 6 ++ helm/cosmo/charts/controlplane/values.yaml | 4 + helm/cosmo/values.yaml | 8 ++ 19 files changed, 200 insertions(+), 58 deletions(-) diff --git a/cdn-server/.env.example b/cdn-server/.env.example index 49b473f69a..65c417d38e 100644 --- a/cdn-server/.env.example +++ b/cdn-server/.env.example @@ -4,4 +4,7 @@ AUTH_ADMISSION_JWT_SECRET="uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW" S3_STORAGE_URL=http://minio:changeme@localhost:10000/cosmo S3_REGION='auto' -S3_ENDPOINT="http://localhost:10000" \ No newline at end of file +S3_ENDPOINT= + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index 78684fba7c..deeb890322 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -91,7 +91,11 @@ export const createS3BlobStorage = (storageUrl: string): BlobStorage => { const endpoint = url.searchParams.get('endpoint') ?? process.env.S3_ENDPOINT; const bucketName = extractS3BucketName(storageUrl); - const s3Config = createS3ClientConfig(storageUrl, bucketName, region, endpoint); + const s3Config = createS3ClientConfig(bucketName, { + url: storageUrl, + region, + endpoint, + }); const s3Client = new S3Client(s3Config); return new S3BlobStorage(s3Client, bucketName); diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts index cd454af89d..455c0c2f5c 100644 --- a/cdn-server/src/utils.ts +++ b/cdn-server/src/utils.ts @@ -1,24 +1,30 @@ import { S3ClientConfig } from '@aws-sdk/client-s3'; +interface S3StorageOptions { + url: string; + region?: string; + endpoint?: string; + username?: string; + password?: string; +} + // see: controlplane/test/utils.s3storage.test.ts -export function createS3ClientConfig( - s3Url: string, - bucketName: string, - region: string | undefined, - endpoint: string | undefined, -): S3ClientConfig { - const url = new URL(s3Url); + +export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { + const url = new URL(opts.url); + const { region, username, password } = opts; const forcePathStyle = !isVirtualHostStyleUrl(url); + const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - const accessKeyId = url.username ?? ''; - const secretAccessKey = url.password ?? ''; + const accessKeyId = url.username || username || ''; + const secretAccessKey = url.password || password || ''; - if (forcePathStyle && !endpoint) { - endpoint = url.origin; + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); } - if (!forcePathStyle && !endpoint) { - endpoint = url.origin.replace(`${bucketName}.`, ''); + if (!region) { + throw new Error('Missing region in S3 configuration.'); } return { diff --git a/controlplane/.env.example b/controlplane/.env.example index a56a3b1034..2388c7814a 100644 --- a/controlplane/.env.example +++ b/controlplane/.env.example @@ -32,6 +32,10 @@ WEBHOOK_SECRET= # S3 S3_STORAGE_URL="http://minio:changeme@localhost:10000/cosmo" +S3_REGION="auto" +S3_ENDPOINT="" +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= # Optional for Stripe Integration DEFAULT_PLAN="developer@1" diff --git a/controlplane/src/core/build-server.ts b/controlplane/src/core/build-server.ts index e5366d1c55..ce2df020d6 100644 --- a/controlplane/src/core/build-server.ts +++ b/controlplane/src/core/build-server.ts @@ -90,6 +90,8 @@ export interface BuildConfig { url: string; endpoint?: string; region?: string; + username?: string; + password?: string; }; mailer: { smtpEnabled: boolean; @@ -352,7 +354,7 @@ export default async function build(opts: BuildConfig) { } const bucketName = extractS3BucketName(opts.s3Storage.url); - const s3Config = createS3ClientConfig(opts.s3Storage.url, bucketName, opts.s3Storage.region, opts.s3Storage.endpoint); + const s3Config = createS3ClientConfig(bucketName, opts.s3Storage); const s3Client = new S3Client(s3Config); const blobStorage = new S3BlobStorage(s3Client, bucketName); diff --git a/controlplane/src/core/env.schema.ts b/controlplane/src/core/env.schema.ts index a65a3fa4c5..2da311694d 100644 --- a/controlplane/src/core/env.schema.ts +++ b/controlplane/src/core/env.schema.ts @@ -119,6 +119,13 @@ export const envVariables = z S3_STORAGE_URL: z.string(), S3_ENDPOINT: z.string().optional(), S3_REGION: z.string().default('auto'), + /** + * Either use: + * https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com + * Or set: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + */ + AWS_ACCESS_KEY_ID: z.string().optional(), + AWS_SECRET_ACCESS_KEY: z.string().optional(), /** * Email diff --git a/controlplane/src/core/util.ts b/controlplane/src/core/util.ts index 9d5a84ca62..7256f5136a 100644 --- a/controlplane/src/core/util.ts +++ b/controlplane/src/core/util.ts @@ -2,6 +2,7 @@ import { randomFill } from 'node:crypto'; import { S3ClientConfig } from '@aws-sdk/client-s3'; import { HandlerContext } from '@connectrpc/connect'; import { + EnumStatusCode, GraphQLSubscriptionProtocol, GraphQLWebsocketSubprotocol, } from '@wundergraph/cosmo-connect/dist/common/common_pb'; @@ -13,7 +14,7 @@ import { uid } from 'uid/secure'; import { AxiosError } from 'axios'; import { isNetworkError, isRetryableError } from 'axios-retry'; import { MemberRole, WebsocketSubprotocol } from '../db/models.js'; -import { AuthContext, DateRange, Label, ResponseMessage } from '../types/index.js'; +import { AuthContext, DateRange, Label, ResponseMessage, S3StorageOptions } from '../types/index.js'; import { isAuthenticationError, isAuthorizationError, isPublicError } from './errors/errors.js'; import { GraphKeyAuthContext } from './services/GraphApiTokenAuthenticator.js'; @@ -373,24 +374,21 @@ export function webhookAxiosRetryCond(err: AxiosError) { return isNetworkError(err) || isRetryableError(err); } -export function createS3ClientConfig( - s3Url: string, - bucketName: string, - region: string | undefined, - endpoint: string | undefined, -): S3ClientConfig { - const url = new URL(s3Url); +export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { + const url = new URL(opts.url); + const { region, username, password } = opts; const forcePathStyle = !isVirtualHostStyleUrl(url); + const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - const accessKeyId = url.username ?? ''; - const secretAccessKey = url.password ?? ''; + const accessKeyId = url.username || username || ''; + const secretAccessKey = url.password || password || ''; - if (forcePathStyle && !endpoint) { - endpoint = url.origin; + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); } - if (!forcePathStyle && !endpoint) { - endpoint = url.origin.replace(`${bucketName}.`, ''); + if (!region) { + throw new Error('Missing region in S3 configuration.'); } return { diff --git a/controlplane/src/types/index.ts b/controlplane/src/types/index.ts index 946e49e49a..08aeacfc58 100644 --- a/controlplane/src/types/index.ts +++ b/controlplane/src/types/index.ts @@ -576,3 +576,11 @@ export interface SchemaLintIssues { warnings: LintIssueResult[]; errors: LintIssueResult[]; } + +export interface S3StorageOptions { + url: string; + region?: string; + endpoint?: string; + username?: string; + password?: string; +} diff --git a/controlplane/test/utils.s3storage.test.ts b/controlplane/test/utils.s3storage.test.ts index 4c928d8810..7919e2a3bd 100644 --- a/controlplane/test/utils.s3storage.test.ts +++ b/controlplane/test/utils.s3storage.test.ts @@ -3,13 +3,17 @@ import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from describe('S3 Utils', () => { describe('createS3ClientConfig', () => { - test('that it correctly configures an S3 client for a path-style URL', () => { - const s3Url = 'http://username:password@minio:9000/cosmo'; + test('correctly configures an S3 client for a path-style URL', () => { const bucketName = 'cosmo'; - const region = 'auto'; - const endpoint = undefined; + const opts = { + url: 'http://username:password@minio:9000/cosmo', + region: 'auto', + endpoint: undefined, + username: '', + password: '', + }; - const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'auto', @@ -22,13 +26,17 @@ describe('S3 Utils', () => { }); }); - test('that it correctly configures an S3 client for a virtual-hosted-style URL with provided endpoint', () => { - const s3Url = 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com'; + test('correctly configures an S3 client for a virtual-hosted-style URL with provided endpoint', () => { const bucketName = 'cosmo-controlplane-bucket'; - const region = 'us-east-1'; - const endpoint = 's3.amazonaws.com'; + const opts = { + url: 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com', + region: 'us-east-1', + endpoint: 's3.amazonaws.com', + username: '', + password: '', + }; - const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'us-east-1', @@ -41,13 +49,17 @@ describe('S3 Utils', () => { }); }); - test('that it correctly configures an S3 client for a virtual-hosted-style URL without provided endpoint', () => { - const s3Url = 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com'; + test('correctly configures an S3 client for a virtual-hosted-style URL without provided endpoint', () => { const bucketName = 'cosmo-controlplane-bucket'; - const region = 'us-east-1'; - const endpoint = undefined; + const opts = { + url: 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com', + region: 'us-east-1', + endpoint: undefined, + username: '', + password: '', + }; - const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'us-east-1', @@ -60,28 +72,60 @@ describe('S3 Utils', () => { }); }); - test('that it handles missing username and password in the URL correctly', () => { - const s3Url = 'https://cosmo-controlplane-bucket.s3.amazonaws.com'; + test('throws an AuthenticationError if credentials are missing', () => { + const bucketName = 'cosmo-controlplane-bucket'; + const opts = { + url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com', + region: 'us-east-1', + endpoint: '', + username: '', + password: '', + }; + + expect(() => createS3ClientConfig(bucketName, opts)).toThrowError( + 'Missing S3 credentials. Please provide access key ID and secret access key.', + ); + }); + + test('correctly configures an S3 client when credentials are provided in opts', () => { const bucketName = 'cosmo-controlplane-bucket'; - const region = 'us-east-1'; - const endpoint = ''; + const opts = { + url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com', + region: 'us-east-1', + endpoint: '', + username: 'testUser', + password: 'testPass', + }; - const config = createS3ClientConfig(s3Url, bucketName, region, endpoint); + const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'us-east-1', endpoint: 'https://s3.amazonaws.com', credentials: { - accessKeyId: '', - secretAccessKey: '', + accessKeyId: 'testUser', + secretAccessKey: 'testPass', }, forcePathStyle: false, }); }); + + test('throws an error if region is missing', () => { + const bucketName = 'cosmo-controlplane-bucket'; + const opts = { + url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com', + region: '', + endpoint: '', + username: 'testUser', + password: 'testPass', + }; + + expect(() => createS3ClientConfig(bucketName, opts)).toThrowError('Missing region in S3 configuration.'); + }); }); describe('extractS3BucketName', () => { - test('that it returns the correct bucket name for a virtual-hosted-style URL', () => { + test('returns the correct bucket name for a virtual-hosted-style URL', () => { const s3Url = 'https://cosmo-controlplane-bucket.s3.amazonaws.com/some/object'; const bucketName = extractS3BucketName(s3Url); @@ -89,7 +133,7 @@ describe('S3 Utils', () => { expect(bucketName).toBe('cosmo-controlplane-bucket'); }); - test('that it returns the correct bucket name for a path-style URL', () => { + test('returns the correct bucket name for a path-style URL', () => { const s3Url = 'http://minio:9000/cosmo'; const bucketName = extractS3BucketName(s3Url); @@ -97,7 +141,7 @@ describe('S3 Utils', () => { expect(bucketName).toBe('cosmo'); }); - test('that it returns the correct bucket name when the URL has multiple path segments', () => { + test('returns the correct bucket name when the URL has multiple path segments', () => { const s3Url = 'http://username:password@localhost:9000/foo'; const bucketName = extractS3BucketName(s3Url); @@ -107,7 +151,7 @@ describe('S3 Utils', () => { }); describe('isVirtualHostStyleUrl', () => { - test('that it returns true for a virtual-hosted-style URL', () => { + test('returns true for a virtual-hosted-style URL', () => { const url = new URL('https://cosmo-controlplane-bucket.s3.amazonaws.com'); const result = isVirtualHostStyleUrl(url); @@ -115,7 +159,7 @@ describe('S3 Utils', () => { expect(result).toBe(true); }); - test('that it returns false for a path-style URL', () => { + test('returns false for a path-style URL', () => { const url = new URL('http://minio:9000/cosmo'); const result = isVirtualHostStyleUrl(url); @@ -123,7 +167,7 @@ describe('S3 Utils', () => { expect(result).toBe(false); }); - test('that it returns false for a custom domain without bucket name in the hostname', () => { + test('returns false for a custom domain without bucket name in the hostname', () => { const url = new URL('https://example.com/cosmo'); const result = isVirtualHostStyleUrl(url); diff --git a/helm/cosmo/README.md b/helm/cosmo/README.md index 00a5d82280..944e6bf38d 100644 --- a/helm/cosmo/README.md +++ b/helm/cosmo/README.md @@ -35,6 +35,8 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | Key | Type | Default | Description | |-----|------|---------|-------------| | cdn.commonLabels | object | `{}` | Add labels to all deployed resources | +| cdn.configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | +| cdn.configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | | cdn.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | | cdn.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | cdn.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | @@ -51,6 +53,8 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | controlplane.commonLabels | object | `{}` | Add labels to all deployed resources | | controlplane.configuration.allowedOrigins[0] | string | `"http://studio.wundergraph.local"` | | | controlplane.configuration.authRedirectUri | string | `"http://controlplane.wundergraph.local/v1/auth/callback"` | | +| controlplane.configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | +| controlplane.configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | | controlplane.configuration.cdnBaseUrl | string | `"http://cosmo-cdn:8787"` | | | controlplane.configuration.clickhouseDsn | string | `"http://default:changeme@cosmo-clickhouse:8123/?database=cosmo"` | | | controlplane.configuration.clickhouseMigrationDsn | string | `"clickhouse://default:changeme@cosmo-clickhouse:9000/cosmo?dial_timeout=15s&max_execution_time=60"` | | diff --git a/helm/cosmo/charts/cdn/README.md b/helm/cosmo/charts/cdn/README.md index 71468cc6e5..523468edad 100644 --- a/helm/cosmo/charts/cdn/README.md +++ b/helm/cosmo/charts/cdn/README.md @@ -16,6 +16,8 @@ WunderGraph Cosmo CDN | autoscaling.minReplicas | int | `1` | | | autoscaling.targetCPUUtilizationPercentage | int | `80` | | | commonLabels | object | `{}` | Add labels to all deployed resources | +| configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | +| configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | | configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | | configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | deploymentStrategy | object | `{}` | | diff --git a/helm/cosmo/charts/cdn/templates/deployment.yaml b/helm/cosmo/charts/cdn/templates/deployment.yaml index 940c65828c..ea972930c9 100644 --- a/helm/cosmo/charts/cdn/templates/deployment.yaml +++ b/helm/cosmo/charts/cdn/templates/deployment.yaml @@ -73,6 +73,22 @@ spec: key: s3Endpoint {{- end }} + {{- if .Values.configuration.awsAccessKeyId }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "cdn.secretName" . }} + key: awsAccessKeyId + {{- end }} + + {{- if .Values.configuration.awsSecretAccessKey }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "cdn.secretName" . }} + key: awsSecretAccessKey + {{- end }} + - name: S3_STORAGE_URL valueFrom: secretKeyRef: diff --git a/helm/cosmo/charts/cdn/templates/secret.yaml b/helm/cosmo/charts/cdn/templates/secret.yaml index 07f8a1b4ca..6f38b22630 100644 --- a/helm/cosmo/charts/cdn/templates/secret.yaml +++ b/helm/cosmo/charts/cdn/templates/secret.yaml @@ -17,4 +17,10 @@ stringData: authJwtSecret: "{{ .Values.global.controlplane.jwtSecret }}" authAdmissionJwtSecret: "{{ .Values.global.controlplane.admissionJwtSecret }}" s3StorageUrl: "{{ .Values.configuration.s3StorageUrl }}" + {{- if .Values.configuration.awsAccessKeyId }} + awsAccessKeyId: "{{ .Values.configuration.awsAccessKeyId }}" + {{- end }} + {{- if .Values.configuration.awsSecretAccessKey }} + awsSecretAccessKey: "{{ .Values.configuration.awsSecretAccessKey }}" + {{- end }} {{- end }} diff --git a/helm/cosmo/charts/cdn/values.yaml b/helm/cosmo/charts/cdn/values.yaml index 80556ff256..aeec4bbadd 100644 --- a/helm/cosmo/charts/cdn/values.yaml +++ b/helm/cosmo/charts/cdn/values.yaml @@ -148,6 +148,10 @@ configuration: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' + # -- Aws access key id, can be used instead of [username]:[password] in the url + awsAccessKeyId: '' + # -- Aws secret access key, can be used instead of [username]:[password] in the url + awsSecretAccessKey: '' # -- Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. existingSecret: "" diff --git a/helm/cosmo/charts/controlplane/README.md b/helm/cosmo/charts/controlplane/README.md index 915deab74d..9c1d9eb561 100644 --- a/helm/cosmo/charts/controlplane/README.md +++ b/helm/cosmo/charts/controlplane/README.md @@ -19,6 +19,8 @@ WunderGraph Cosmo Controlplane | commonLabels | object | `{}` | Add labels to all deployed resources | | configuration.allowedOrigins[0] | string | `"*"` | | | configuration.authRedirectUri | string | `"http://controlplane.wundergraph.local/v1/auth/callback"` | | +| configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | +| configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | | configuration.cdnBaseUrl | string | `"http://cosmo-cdn:8787"` | URL of the CDN to use for serving router configs and persistent operations | | configuration.clickhouseDsn | string | `"http://default:changeme@cosmo-clickhouse:8123?database=cosmo"` | | | configuration.clickhouseMigrationDsn | string | `"clickhouse://default:changeme@cosmo-clickhouse:9000?database=cosmo"` | | diff --git a/helm/cosmo/charts/controlplane/templates/deployment.yaml b/helm/cosmo/charts/controlplane/templates/deployment.yaml index 313a5631e7..b1d5538695 100644 --- a/helm/cosmo/charts/controlplane/templates/deployment.yaml +++ b/helm/cosmo/charts/controlplane/templates/deployment.yaml @@ -238,6 +238,20 @@ spec: name: {{ include "controlplane.fullname" . }}-configmap key: s3Endpoint {{- end }} + {{- if .Values.configuration.awsAccessKeyId }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "controlplane.secretName" . }} + key: awsAccessKeyId + {{- end }} + {{- if .Values.configuration.awsSecretAccessKey }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "controlplane.secretName" . }} + key: awsSecretAccessKey + {{- end }} - name: SMTP_ENABLED valueFrom: configMapKeyRef: diff --git a/helm/cosmo/charts/controlplane/templates/secret.yaml b/helm/cosmo/charts/controlplane/templates/secret.yaml index 3e201e14ae..fa53a3cacc 100644 --- a/helm/cosmo/charts/controlplane/templates/secret.yaml +++ b/helm/cosmo/charts/controlplane/templates/secret.yaml @@ -42,4 +42,10 @@ stringData: smtpUsername: "{{ .Values.configuration.smtp.username }}" smtpPassword: "{{ .Values.configuration.smtp.password }}" {{- end }} + {{- if .Values.configuration.awsAccessKeyId }} + awsAccessKeyId: "{{ .Values.configuration.awsAccessKeyId }}" + {{- end }} + {{- if .Values.configuration.awsSecretAccessKey }} + awsSecretAccessKey: "{{ .Values.configuration.awsSecretAccessKey }}" + {{- end }} {{- end }} diff --git a/helm/cosmo/charts/controlplane/values.yaml b/helm/cosmo/charts/controlplane/values.yaml index fe23642156..d97d594595 100644 --- a/helm/cosmo/charts/controlplane/values.yaml +++ b/helm/cosmo/charts/controlplane/values.yaml @@ -188,6 +188,10 @@ configuration: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' + # -- Aws access key id, can be used instead of [username]:[password] in the url + awsAccessKeyId: '' + # -- Aws secret access key, can be used instead of [username]:[password] in the url + awsSecretAccessKey: '' stripeSecretKey: '' stripeWebhookSecret: '' # -- The default billing plan, eg `developer@1` diff --git a/helm/cosmo/values.yaml b/helm/cosmo/values.yaml index 5bb3c4b1a6..2de8f61849 100644 --- a/helm/cosmo/values.yaml +++ b/helm/cosmo/values.yaml @@ -124,6 +124,10 @@ cdn: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' + # -- Aws access key id, can be used instead of [username]:[password] in the url + awsAccessKeyId: '' + # -- Aws secret access key, can be used instead of [username]:[password] in the url + awsSecretAccessKey: '' # Cosmo Controlplane. For more options, please refer to the README.md controlplane: @@ -194,6 +198,10 @@ controlplane: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' + # -- Aws access key id, can be used instead of [username]:[password] in the url + awsAccessKeyId: '' + # -- Aws secret access key, can be used instead of [username]:[password] in the url + awsSecretAccessKey: '' cdnBaseUrl: 'http://cosmo-cdn:8787' From 1287a5b5e3f9e6d1a657e1f9bfef6bc58be83d5d Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 11:20:01 +0200 Subject: [PATCH 10/20] refactor: do not use aws namings to avoid shadowings --- cdn-server/.env.example | 5 ++--- cdn-server/src/s3.ts | 4 ++++ controlplane/.env.example | 4 ++-- controlplane/src/core/env.schema.ts | 6 +++--- controlplane/src/index.ts | 4 ++++ helm/cosmo/README.md | 8 ++++---- helm/cosmo/charts/cdn/README.md | 4 ++-- helm/cosmo/charts/cdn/templates/deployment.yaml | 12 ++++++------ helm/cosmo/charts/cdn/templates/secret.yaml | 8 ++++---- helm/cosmo/charts/cdn/values.yaml | 8 ++++---- helm/cosmo/charts/controlplane/README.md | 4 ++-- .../controlplane/templates/deployment.yaml | 12 ++++++------ .../charts/controlplane/templates/secret.yaml | 8 ++++---- helm/cosmo/charts/controlplane/values.yaml | 8 ++++---- helm/cosmo/values.yaml | 16 ++++++++-------- 15 files changed, 59 insertions(+), 52 deletions(-) diff --git a/cdn-server/.env.example b/cdn-server/.env.example index 65c417d38e..4b1d442213 100644 --- a/cdn-server/.env.example +++ b/cdn-server/.env.example @@ -5,6 +5,5 @@ AUTH_ADMISSION_JWT_SECRET="uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW" S3_STORAGE_URL=http://minio:changeme@localhost:10000/cosmo S3_REGION='auto' S3_ENDPOINT= - -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index deeb890322..eff41947a9 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -89,12 +89,16 @@ export const createS3BlobStorage = (storageUrl: string): BlobStorage => { const url = new URL(storageUrl); const region = url.searchParams.get('region') ?? process.env.S3_REGION ?? 'default'; const endpoint = url.searchParams.get('endpoint') ?? process.env.S3_ENDPOINT; + const username = process.env.S3_ACCESS_KEY_ID || ''; + const password = process.env.S3_SECRET_ACCESS_KEY || ''; const bucketName = extractS3BucketName(storageUrl); const s3Config = createS3ClientConfig(bucketName, { url: storageUrl, region, endpoint, + username, + password, }); const s3Client = new S3Client(s3Config); diff --git a/controlplane/.env.example b/controlplane/.env.example index 2388c7814a..75f4f36ffb 100644 --- a/controlplane/.env.example +++ b/controlplane/.env.example @@ -34,8 +34,8 @@ WEBHOOK_SECRET= S3_STORAGE_URL="http://minio:changeme@localhost:10000/cosmo" S3_REGION="auto" S3_ENDPOINT="" -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= # Optional for Stripe Integration DEFAULT_PLAN="developer@1" diff --git a/controlplane/src/core/env.schema.ts b/controlplane/src/core/env.schema.ts index 2da311694d..732a3ae5b5 100644 --- a/controlplane/src/core/env.schema.ts +++ b/controlplane/src/core/env.schema.ts @@ -122,10 +122,10 @@ export const envVariables = z /** * Either use: * https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com - * Or set: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + * Or set: S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY */ - AWS_ACCESS_KEY_ID: z.string().optional(), - AWS_SECRET_ACCESS_KEY: z.string().optional(), + S3_ACCESS_KEY_ID: z.string().optional(), + S3_SECRET_ACCESS_KEY: z.string().optional(), /** * Email diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index fcefaefdf0..d7aba46ab4 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -43,6 +43,8 @@ const { S3_STORAGE_URL, S3_ENDPOINT, S3_REGION, + S3_ACCESS_KEY_ID, + S3_SECRET_ACCESS_KEY, SMTP_ENABLED, SMTP_HOST, SMTP_PORT, @@ -122,6 +124,8 @@ const options: BuildConfig = { url: S3_STORAGE_URL, region: S3_REGION, endpoint: S3_ENDPOINT, + username: S3_ACCESS_KEY_ID, + password: S3_SECRET_ACCESS_KEY, }, mailer: { smtpEnabled: SMTP_ENABLED, diff --git a/helm/cosmo/README.md b/helm/cosmo/README.md index 944e6bf38d..52d22b6eed 100644 --- a/helm/cosmo/README.md +++ b/helm/cosmo/README.md @@ -35,10 +35,10 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | Key | Type | Default | Description | |-----|------|---------|-------------| | cdn.commonLabels | object | `{}` | Add labels to all deployed resources | -| cdn.configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | -| cdn.configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | +| cdn.configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | cdn.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | | cdn.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | +| cdn.configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | cdn.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | | clickhouse.auth.password | string | `"changeme"` | | | clickhouse.auth.username | string | `"default"` | | @@ -53,8 +53,6 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | controlplane.commonLabels | object | `{}` | Add labels to all deployed resources | | controlplane.configuration.allowedOrigins[0] | string | `"http://studio.wundergraph.local"` | | | controlplane.configuration.authRedirectUri | string | `"http://controlplane.wundergraph.local/v1/auth/callback"` | | -| controlplane.configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | -| controlplane.configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | | controlplane.configuration.cdnBaseUrl | string | `"http://cosmo-cdn:8787"` | | | controlplane.configuration.clickhouseDsn | string | `"http://default:changeme@cosmo-clickhouse:8123/?database=cosmo"` | | | controlplane.configuration.clickhouseMigrationDsn | string | `"clickhouse://default:changeme@cosmo-clickhouse:9000/cosmo?dial_timeout=15s&max_execution_time=60"` | | @@ -70,8 +68,10 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | controlplane.configuration.prometheus.port | int | `8088` | The port where metrics are exposed. Default is port 8088. | | controlplane.configuration.redisHost | string | `"cosmo-redis-master"` | | | controlplane.configuration.redisPort | int | `6379` | | +| controlplane.configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | controlplane.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | | controlplane.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | +| controlplane.configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | controlplane.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | | controlplane.configuration.smtp | object | `{"enabled":false,"host":"smtp.postmarkapp.com","password":"","port":587,"requireTls":true,"secure":true,"username":""}` | Use this section to configure the smtp server. | | controlplane.configuration.smtp.enabled | bool | `false` | Enables the smtp server. Default is false. | diff --git a/helm/cosmo/charts/cdn/README.md b/helm/cosmo/charts/cdn/README.md index 523468edad..da1d99c3a1 100644 --- a/helm/cosmo/charts/cdn/README.md +++ b/helm/cosmo/charts/cdn/README.md @@ -16,10 +16,10 @@ WunderGraph Cosmo CDN | autoscaling.minReplicas | int | `1` | | | autoscaling.targetCPUUtilizationPercentage | int | `80` | | | commonLabels | object | `{}` | Add labels to all deployed resources | -| configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | -| configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | +| configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | | configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | +| configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | deploymentStrategy | object | `{}` | | | existingSecret | string | `""` | Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. | | extraEnvVars | list | `[]` | Allows to set additional environment variables on the container. Useful for global application non-specific settings. | diff --git a/helm/cosmo/charts/cdn/templates/deployment.yaml b/helm/cosmo/charts/cdn/templates/deployment.yaml index ea972930c9..3edfdbf935 100644 --- a/helm/cosmo/charts/cdn/templates/deployment.yaml +++ b/helm/cosmo/charts/cdn/templates/deployment.yaml @@ -73,20 +73,20 @@ spec: key: s3Endpoint {{- end }} - {{- if .Values.configuration.awsAccessKeyId }} - - name: AWS_ACCESS_KEY_ID + {{- if .Values.configuration.s3AccessKeyId }} + - name: S3_ACCESS_KEY_ID valueFrom: secretKeyRef: name: {{ include "cdn.secretName" . }} - key: awsAccessKeyId + key: s3AccessKeyId {{- end }} - {{- if .Values.configuration.awsSecretAccessKey }} - - name: AWS_SECRET_ACCESS_KEY + {{- if .Values.configuration.s3SecretAccessKey }} + - name: S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: {{ include "cdn.secretName" . }} - key: awsSecretAccessKey + key: s3SecretAccessKey {{- end }} - name: S3_STORAGE_URL diff --git a/helm/cosmo/charts/cdn/templates/secret.yaml b/helm/cosmo/charts/cdn/templates/secret.yaml index 6f38b22630..b0549bc75a 100644 --- a/helm/cosmo/charts/cdn/templates/secret.yaml +++ b/helm/cosmo/charts/cdn/templates/secret.yaml @@ -17,10 +17,10 @@ stringData: authJwtSecret: "{{ .Values.global.controlplane.jwtSecret }}" authAdmissionJwtSecret: "{{ .Values.global.controlplane.admissionJwtSecret }}" s3StorageUrl: "{{ .Values.configuration.s3StorageUrl }}" - {{- if .Values.configuration.awsAccessKeyId }} - awsAccessKeyId: "{{ .Values.configuration.awsAccessKeyId }}" + {{- if .Values.configuration.s3AccessKeyId }} + s3AccessKeyId: "{{ .Values.configuration.s3AccessKeyId }}" {{- end }} - {{- if .Values.configuration.awsSecretAccessKey }} - awsSecretAccessKey: "{{ .Values.configuration.awsSecretAccessKey }}" + {{- if .Values.configuration.s3SecretAccessKey }} + s3SecretAccessKey: "{{ .Values.configuration.s3SecretAccessKey }}" {{- end }} {{- end }} diff --git a/helm/cosmo/charts/cdn/values.yaml b/helm/cosmo/charts/cdn/values.yaml index aeec4bbadd..4d7cb54f1f 100644 --- a/helm/cosmo/charts/cdn/values.yaml +++ b/helm/cosmo/charts/cdn/values.yaml @@ -148,10 +148,10 @@ configuration: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' - # -- Aws access key id, can be used instead of [username]:[password] in the url - awsAccessKeyId: '' - # -- Aws secret access key, can be used instead of [username]:[password] in the url - awsSecretAccessKey: '' + # -- s3 access key id, can be used instead of [username]:[password] in the url + s3AccessKeyId: '' + # -- s3 secret access key, can be used instead of [username]:[password] in the url + s3SecretAccessKey: '' # -- Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. existingSecret: "" diff --git a/helm/cosmo/charts/controlplane/README.md b/helm/cosmo/charts/controlplane/README.md index 9c1d9eb561..3abeb5c2db 100644 --- a/helm/cosmo/charts/controlplane/README.md +++ b/helm/cosmo/charts/controlplane/README.md @@ -19,8 +19,6 @@ WunderGraph Cosmo Controlplane | commonLabels | object | `{}` | Add labels to all deployed resources | | configuration.allowedOrigins[0] | string | `"*"` | | | configuration.authRedirectUri | string | `"http://controlplane.wundergraph.local/v1/auth/callback"` | | -| configuration.awsAccessKeyId | string | `""` | Aws access key id, can be used instead of [username]:[password] in the url | -| configuration.awsSecretAccessKey | string | `""` | Aws secret access key, can be used instead of [username]:[password] in the url | | configuration.cdnBaseUrl | string | `"http://cosmo-cdn:8787"` | URL of the CDN to use for serving router configs and persistent operations | | configuration.clickhouseDsn | string | `"http://default:changeme@cosmo-clickhouse:8123?database=cosmo"` | | | configuration.clickhouseMigrationDsn | string | `"clickhouse://default:changeme@cosmo-clickhouse:9000?database=cosmo"` | | @@ -51,8 +49,10 @@ WunderGraph Cosmo Controlplane | configuration.redisTlsCa | string | `""` | When connecting to a redis instance over TLS. Accept a cert in PEM format (as one-line with \n) or file. | | configuration.redisTlsCert | string | `""` | | | configuration.redisTlsKey | string | `""` | | +| configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | | configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | +| configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | | configuration.slackAppClientId | string | `""` | | | configuration.slackAppClientSecret | string | `""` | | diff --git a/helm/cosmo/charts/controlplane/templates/deployment.yaml b/helm/cosmo/charts/controlplane/templates/deployment.yaml index b1d5538695..22364231aa 100644 --- a/helm/cosmo/charts/controlplane/templates/deployment.yaml +++ b/helm/cosmo/charts/controlplane/templates/deployment.yaml @@ -238,19 +238,19 @@ spec: name: {{ include "controlplane.fullname" . }}-configmap key: s3Endpoint {{- end }} - {{- if .Values.configuration.awsAccessKeyId }} - - name: AWS_ACCESS_KEY_ID + {{- if .Values.configuration.s3AccessKeyId }} + - name: S3_ACCESS_KEY_ID valueFrom: secretKeyRef: name: {{ include "controlplane.secretName" . }} - key: awsAccessKeyId + key: s3AccessKeyId {{- end }} - {{- if .Values.configuration.awsSecretAccessKey }} - - name: AWS_SECRET_ACCESS_KEY + {{- if .Values.configuration.s3SecretAccessKey }} + - name: S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: {{ include "controlplane.secretName" . }} - key: awsSecretAccessKey + key: s3SecretAccessKey {{- end }} - name: SMTP_ENABLED valueFrom: diff --git a/helm/cosmo/charts/controlplane/templates/secret.yaml b/helm/cosmo/charts/controlplane/templates/secret.yaml index fa53a3cacc..1bba9daf65 100644 --- a/helm/cosmo/charts/controlplane/templates/secret.yaml +++ b/helm/cosmo/charts/controlplane/templates/secret.yaml @@ -42,10 +42,10 @@ stringData: smtpUsername: "{{ .Values.configuration.smtp.username }}" smtpPassword: "{{ .Values.configuration.smtp.password }}" {{- end }} - {{- if .Values.configuration.awsAccessKeyId }} - awsAccessKeyId: "{{ .Values.configuration.awsAccessKeyId }}" + {{- if .Values.configuration.s3AccessKeyId }} + s3AccessKeyId: "{{ .Values.configuration.s3AccessKeyId }}" {{- end }} - {{- if .Values.configuration.awsSecretAccessKey }} - awsSecretAccessKey: "{{ .Values.configuration.awsSecretAccessKey }}" + {{- if .Values.configuration.s3SecretAccessKey }} + s3SecretAccessKey: "{{ .Values.configuration.s3SecretAccessKey }}" {{- end }} {{- end }} diff --git a/helm/cosmo/charts/controlplane/values.yaml b/helm/cosmo/charts/controlplane/values.yaml index d97d594595..5a8645af9b 100644 --- a/helm/cosmo/charts/controlplane/values.yaml +++ b/helm/cosmo/charts/controlplane/values.yaml @@ -188,10 +188,10 @@ configuration: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' - # -- Aws access key id, can be used instead of [username]:[password] in the url - awsAccessKeyId: '' - # -- Aws secret access key, can be used instead of [username]:[password] in the url - awsSecretAccessKey: '' + # -- s3 access key id, can be used instead of [username]:[password] in the url + s3AccessKeyId: '' + # -- s3 secret access key, can be used instead of [username]:[password] in the url + s3SecretAccessKey: '' stripeSecretKey: '' stripeWebhookSecret: '' # -- The default billing plan, eg `developer@1` diff --git a/helm/cosmo/values.yaml b/helm/cosmo/values.yaml index 2de8f61849..d9ed8b3f36 100644 --- a/helm/cosmo/values.yaml +++ b/helm/cosmo/values.yaml @@ -124,10 +124,10 @@ cdn: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' - # -- Aws access key id, can be used instead of [username]:[password] in the url - awsAccessKeyId: '' - # -- Aws secret access key, can be used instead of [username]:[password] in the url - awsSecretAccessKey: '' + # -- s3 access key id, can be used instead of [username]:[password] in the url + s3AccessKeyId: '' + # -- s3 secret access key, can be used instead of [username]:[password] in the url + s3SecretAccessKey: '' # Cosmo Controlplane. For more options, please refer to the README.md controlplane: @@ -198,10 +198,10 @@ controlplane: s3Region: 'auto' # -- The endpoint of the S3 bucket. s3Endpoint: '' - # -- Aws access key id, can be used instead of [username]:[password] in the url - awsAccessKeyId: '' - # -- Aws secret access key, can be used instead of [username]:[password] in the url - awsSecretAccessKey: '' + # -- s3 access key id, can be used instead of [username]:[password] in the url + s3AccessKeyId: '' + # -- s3 secret access key, can be used instead of [username]:[password] in the url + s3SecretAccessKey: '' cdnBaseUrl: 'http://cosmo-cdn:8787' From 83851608d5b282c5783514f6fcab515a91fe605b Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 11:51:01 +0200 Subject: [PATCH 11/20] chore: use right env var for controlplane region --- docker-compose.full.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.full.yml b/docker-compose.full.yml index aeb65930c0..b1875199cc 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -274,6 +274,7 @@ services: PROMETHEUS_API_URL: 'http://admin:test@prometheus:9090/api/v1' S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo} S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} + S3_REGION: ${S3_REGION_CONTROLPLANE:-${S3_REGION:-auto}} CDN_BASE_URL: 'http://cdn:11000' REDIS_HOST: redis REDIS_PORT: 6379 From d6a5228b2be500e4d6a0eb90db0981652b34ac4c Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 12:00:09 +0200 Subject: [PATCH 12/20] chore: add backwards compatible env vars --- docker-compose.full.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.full.yml b/docker-compose.full.yml index b1875199cc..4ab2fb913c 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -211,6 +211,8 @@ services: AUTH_ADMISSION_JWT_SECRET: uXDxJLEvrw4aafPfrf3rRotCoBzRfPEW S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo} S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} ports: - '11000:11000' networks: @@ -273,8 +275,9 @@ services: KC_FRONTEND_URL: 'http://localhost:8080' PROMETHEUS_API_URL: 'http://admin:test@prometheus:9090/api/v1' S3_STORAGE_URL: ${S3_STORAGE_URL:-http://${MINIO_ROOT_USER:-minio}:${MINIO_ROOT_PASSWORD:-changeme}@minio:9000/cosmo} - S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} S3_REGION: ${S3_REGION_CONTROLPLANE:-${S3_REGION:-auto}} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} CDN_BASE_URL: 'http://cdn:11000' REDIS_HOST: redis REDIS_PORT: 6379 From 14c321ffcfd6f663b8b1ad5e24153b273f332b25 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 12:16:36 +0200 Subject: [PATCH 13/20] docs: update comment --- cdn-server/src/utils.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts index 455c0c2f5c..e0ea06f2a5 100644 --- a/cdn-server/src/utils.ts +++ b/cdn-server/src/utils.ts @@ -1,5 +1,11 @@ import { S3ClientConfig } from '@aws-sdk/client-s3'; +/** + * controlplane and cdn are using the same code for handling the s3 storage. + * + * see: controlplane/test/utils.s3storage.test.ts for further details + */ + interface S3StorageOptions { url: string; region?: string; @@ -8,8 +14,6 @@ interface S3StorageOptions { password?: string; } -// see: controlplane/test/utils.s3storage.test.ts - export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { const url = new URL(opts.url); const { region, username, password } = opts; From 0b382ff0f38fc8d6b9faa9dd1272f18a30724a94 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 12:24:15 +0200 Subject: [PATCH 14/20] docs: update comment --- controlplane/src/core/env.schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controlplane/src/core/env.schema.ts b/controlplane/src/core/env.schema.ts index 732a3ae5b5..c59c05294b 100644 --- a/controlplane/src/core/env.schema.ts +++ b/controlplane/src/core/env.schema.ts @@ -114,7 +114,7 @@ export const envVariables = z * AWS S3 Storage * S3_STORAGE_URL="https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com" * S3_REGION="us-east-1" # set this for amazon to your region - * S3_ENDPOINT="s3.amazonaws.com" # replaces the bucket from the S3_STORAGE_URL origin + * S3_ENDPOINT="s3.amazonaws.com" # replaces the bucket from the S3_STORAGE_URL origin or set it manually to s3.amazonaws.com */ S3_STORAGE_URL: z.string(), S3_ENDPOINT: z.string().optional(), From 5c6d6a6e0e76dd7507dfe890be2505ea287e7e4a Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Mon, 2 Sep 2024 12:34:58 +0200 Subject: [PATCH 15/20] chore: add username and password handling --- controlplane/test/authentication.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controlplane/test/authentication.test.ts b/controlplane/test/authentication.test.ts index 6ee21afa6d..7e7cf3b4a7 100644 --- a/controlplane/test/authentication.test.ts +++ b/controlplane/test/authentication.test.ts @@ -52,6 +52,8 @@ describe('Authentication', (ctx) => { url: 'http://localhost:9000', region: 'auto', endpoint: 'localhost:9000', + username: 'minio', + password: 'changeme', }, mailer: { smtpEnabled: false, From ec1ac055cdd9c2e5deb264a9c691c84fd54e9625 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Tue, 3 Sep 2024 11:55:20 +0200 Subject: [PATCH 16/20] refactor: move s3 config handling to shared module --- cdn-server/package.json | 1 + cdn-server/src/s3.ts | 2 +- cdn-server/src/utils.ts | 58 ------------------- controlplane/src/core/build-server.ts | 3 +- controlplane/src/core/util.ts | 46 +-------------- controlplane/src/types/index.ts | 8 --- pnpm-lock.yaml | 35 +++++------ shared/package.json | 1 + shared/src/types/index.ts | 7 +++ shared/src/utils/util.ts | 45 ++++++++++++++ .../test/utils.s3storage.test.ts | 2 +- 11 files changed, 72 insertions(+), 136 deletions(-) delete mode 100644 cdn-server/src/utils.ts create mode 100644 shared/src/types/index.ts rename {controlplane => shared}/test/utils.s3storage.test.ts (99%) diff --git a/cdn-server/package.json b/cdn-server/package.json index 5dc762812c..3220708906 100644 --- a/cdn-server/package.json +++ b/cdn-server/package.json @@ -25,6 +25,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.529.1", "@hono/node-server": "1.11.0", + "@wundergraph/cosmo-shared": "workspace:*", "@wundergraph/cosmo-cdn": "workspace:*", "dotenv": "^16.4.5", "hono": "4.2.7" diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index eff41947a9..ba4ae54e97 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -1,7 +1,7 @@ import { GetObjectCommand, HeadObjectCommand, NoSuchKey, NotFound, S3Client } from '@aws-sdk/client-s3'; +import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared'; import { BlobNotFoundError, BlobObject, BlobStorage } from '@wundergraph/cosmo-cdn'; import { Context } from 'hono'; -import { createS3ClientConfig, extractS3BucketName } from './utils'; /** * Retrieves objects from S3 given an S3Client and a bucket name diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts deleted file mode 100644 index e0ea06f2a5..0000000000 --- a/cdn-server/src/utils.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { S3ClientConfig } from '@aws-sdk/client-s3'; - -/** - * controlplane and cdn are using the same code for handling the s3 storage. - * - * see: controlplane/test/utils.s3storage.test.ts for further details - */ - -interface S3StorageOptions { - url: string; - region?: string; - endpoint?: string; - username?: string; - password?: string; -} - -export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { - const url = new URL(opts.url); - const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); - const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - - const accessKeyId = url.username || username || ''; - const secretAccessKey = url.password || password || ''; - - if (!accessKeyId || !secretAccessKey) { - throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); - } - - if (!region) { - throw new Error('Missing region in S3 configuration.'); - } - - return { - region, - endpoint, - credentials: { - accessKeyId, - secretAccessKey, - }, - forcePathStyle, - }; -} - -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); - - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; - } - - // path based style - return url.pathname.slice(1); -} - -export function isVirtualHostStyleUrl(url: URL) { - return url.hostname.split('.').length > 2; -} diff --git a/controlplane/src/core/build-server.ts b/controlplane/src/core/build-server.ts index ce2df020d6..c932d0cc21 100644 --- a/controlplane/src/core/build-server.ts +++ b/controlplane/src/core/build-server.ts @@ -2,6 +2,7 @@ import Fastify, { FastifyBaseLogger } from 'fastify'; import { S3Client } from '@aws-sdk/client-s3'; import { fastifyConnectPlugin } from '@connectrpc/connect-fastify'; import { cors, createContextValues } from '@connectrpc/connect'; +import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared'; import fastifyCors from '@fastify/cors'; import { pino, stdTimeFunctions, LoggerOptions } from 'pino'; import { compressionBrotli, compressionGzip } from '@connectrpc/connect-node'; @@ -37,7 +38,7 @@ import { BillingRepository } from './repositories/BillingRepository.js'; import { BillingService } from './services/BillingService.js'; import { UserRepository } from './repositories/UserRepository.js'; import { AIGraphReadmeQueue, createAIGraphReadmeWorker } from './workers/AIGraphReadmeWorker.js'; -import { fastifyLoggerId, createS3ClientConfig, extractS3BucketName } from './util.js'; +import { fastifyLoggerId } from './util.js'; import { ApiKeyRepository } from './repositories/ApiKeyRepository.js'; import { createDeleteOrganizationWorker, DeleteOrganizationQueue } from './workers/DeleteOrganizationWorker.js'; diff --git a/controlplane/src/core/util.ts b/controlplane/src/core/util.ts index 7256f5136a..dcfe82b47e 100644 --- a/controlplane/src/core/util.ts +++ b/controlplane/src/core/util.ts @@ -1,5 +1,4 @@ import { randomFill } from 'node:crypto'; -import { S3ClientConfig } from '@aws-sdk/client-s3'; import { HandlerContext } from '@connectrpc/connect'; import { EnumStatusCode, @@ -14,7 +13,7 @@ import { uid } from 'uid/secure'; import { AxiosError } from 'axios'; import { isNetworkError, isRetryableError } from 'axios-retry'; import { MemberRole, WebsocketSubprotocol } from '../db/models.js'; -import { AuthContext, DateRange, Label, ResponseMessage, S3StorageOptions } from '../types/index.js'; +import { AuthContext, DateRange, Label, ResponseMessage } from '../types/index.js'; import { isAuthenticationError, isAuthorizationError, isPublicError } from './errors/errors.js'; import { GraphKeyAuthContext } from './services/GraphApiTokenAuthenticator.js'; @@ -373,46 +372,3 @@ export function getValueOrDefault(map: Map, key: K, constructor: () export function webhookAxiosRetryCond(err: AxiosError) { return isNetworkError(err) || isRetryableError(err); } - -export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { - const url = new URL(opts.url); - const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); - const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - - const accessKeyId = url.username || username || ''; - const secretAccessKey = url.password || password || ''; - - if (!accessKeyId || !secretAccessKey) { - throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); - } - - if (!region) { - throw new Error('Missing region in S3 configuration.'); - } - - return { - region, - endpoint, - credentials: { - accessKeyId, - secretAccessKey, - }, - forcePathStyle, - }; -} - -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); - - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; - } - - // path based style - return url.pathname.slice(1); -} - -export function isVirtualHostStyleUrl(url: URL) { - return url.hostname.split('.').length > 2; -} diff --git a/controlplane/src/types/index.ts b/controlplane/src/types/index.ts index 08aeacfc58..946e49e49a 100644 --- a/controlplane/src/types/index.ts +++ b/controlplane/src/types/index.ts @@ -576,11 +576,3 @@ export interface SchemaLintIssues { warnings: LintIssueResult[]; errors: LintIssueResult[]; } - -export interface S3StorageOptions { - url: string; - region?: string; - endpoint?: string; - username?: string; - password?: string; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad0f0a8e71..7d3beaf1f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,9 @@ importers: '@wundergraph/cosmo-cdn': specifier: workspace:* version: link:cdn + '@wundergraph/cosmo-shared': + specifier: workspace:* + version: link:../shared dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -703,6 +706,9 @@ importers: shared: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.529.1 + version: 3.529.1 '@bufbuild/protobuf': specifier: ^1.9.0 version: 1.9.0 @@ -1107,7 +1113,7 @@ packages: resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==} dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.433.0 + '@aws-sdk/types': 3.523.0 tslib: 1.14.1 dev: false @@ -1115,7 +1121,7 @@ packages: resolution: {integrity: sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==} dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.433.0 + '@aws-sdk/types': 3.523.0 tslib: 1.14.1 dev: false @@ -1167,7 +1173,7 @@ packages: /@aws-crypto/util@3.0.0: resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==} dependencies: - '@aws-sdk/types': 3.433.0 + '@aws-sdk/types': 3.523.0 '@aws-sdk/util-utf8-browser': 3.259.0 tslib: 1.14.1 dev: false @@ -1663,14 +1669,6 @@ packages: - aws-crt dev: false - /@aws-sdk/types@3.433.0: - resolution: {integrity: sha512-0jEE2mSrNDd8VGFjTc1otYrwYPIkzZJEIK90ZxisKvQ/EURGBhNzWn7ejWB9XCMFT6XumYLBR0V9qq5UPisWtA==} - engines: {node: '>=14.0.0'} - dependencies: - '@smithy/types': 2.4.0 - tslib: 2.6.2 - dev: false - /@aws-sdk/types@3.523.0: resolution: {integrity: sha512-AqGIu4u+SxPiUuNBp2acCVcq80KDUFjxe6e3cMTvKWTzCbrVk1AXv0dAaJnCmdkWIha6zJDWxpIk/aL4EGhZ9A==} engines: {node: '>=14.0.0'} @@ -11544,13 +11542,6 @@ packages: tslib: 2.6.2 dev: false - /@smithy/types@2.4.0: - resolution: {integrity: sha512-iH1Xz68FWlmBJ9vvYeHifVMWJf82ONx+OybPW8ZGf5wnEv2S0UXcU4zwlwJkRXuLKpcSLHrraHbn2ucdVXLb4g==} - engines: {node: '>=14.0.0'} - dependencies: - tslib: 2.6.2 - dev: false - /@smithy/url-parser@2.1.3: resolution: {integrity: sha512-X1NRA4WzK/ihgyzTpeGvI9Wn45y8HmqF4AZ/FazwAv8V203Ex+4lXqcYI70naX9ETqbqKVzFk88W6WJJzCggTQ==} dependencies: @@ -12692,7 +12683,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/type-utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) '@typescript-eslint/utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.52.0 graphemer: 1.4.0 ignore: 5.2.4 @@ -12793,7 +12784,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.5.2) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.52.0 typescript: 5.5.2 transitivePeerDependencies: @@ -12860,7 +12851,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.5.2) '@typescript-eslint/utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.52.0 tsutils: 3.21.0(typescript@5.5.2) typescript: 5.5.2 @@ -16206,7 +16197,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.5 + debug: 4.3.4 enhanced-resolve: 5.15.0 eslint: 8.52.0 eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0)(eslint-import-resolver-typescript@3.5.5)(eslint@8.52.0) diff --git a/shared/package.json b/shared/package.json index bdf24028af..b78f2b4680 100644 --- a/shared/package.json +++ b/shared/package.json @@ -33,6 +33,7 @@ "license": "Apache-2.0", "dependencies": { "@bufbuild/protobuf": "^1.9.0", + "@aws-sdk/client-s3": "^3.529.1", "@graphql-tools/schema": "^8.5.1", "@graphql-tools/utils": "^9.2.1", "@wundergraph/composition": "workspace:*", diff --git a/shared/src/types/index.ts b/shared/src/types/index.ts new file mode 100644 index 0000000000..bce07f22fe --- /dev/null +++ b/shared/src/types/index.ts @@ -0,0 +1,7 @@ +export interface S3StorageOptions { + url: string; + region?: string; + endpoint?: string; + username?: string; + password?: string; +} diff --git a/shared/src/utils/util.ts b/shared/src/utils/util.ts index 9acb0f9e3e..f22a701141 100644 --- a/shared/src/utils/util.ts +++ b/shared/src/utils/util.ts @@ -1,4 +1,6 @@ +import * as S3 from '@aws-sdk/client-s3'; import { SubscriptionProtocol, WebsocketSubprotocol } from '../router-config/builder.js'; +import { S3StorageOptions } from '../types/index.js'; export function delay(t: number) { return new Promise((resolve) => setTimeout(resolve, t)); @@ -76,3 +78,46 @@ export function isValidWebsocketSubprotocol(protocol: WebsocketSubprotocol) { } } } + +export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3.S3ClientConfig { + const url = new URL(opts.url); + const { region, username, password } = opts; + const forcePathStyle = !isVirtualHostStyleUrl(url); + const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); + + const accessKeyId = url.username || username || ''; + const secretAccessKey = url.password || password || ''; + + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); + } + + if (!region) { + throw new Error('Missing region in S3 configuration.'); + } + + return { + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle, + }; +} + +export function extractS3BucketName(s3Url: string) { + const url = new URL(s3Url); + + if (isVirtualHostStyleUrl(url)) { + return url.hostname.split('.')[0]; + } + + // path based style + return url.pathname.slice(1); +} + +export function isVirtualHostStyleUrl(url: URL) { + return url.hostname.split('.').length > 2; +} diff --git a/controlplane/test/utils.s3storage.test.ts b/shared/test/utils.s3storage.test.ts similarity index 99% rename from controlplane/test/utils.s3storage.test.ts rename to shared/test/utils.s3storage.test.ts index 7919e2a3bd..d197619297 100644 --- a/controlplane/test/utils.s3storage.test.ts +++ b/shared/test/utils.s3storage.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src/core/util.js'; +import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src'; describe('S3 Utils', () => { describe('createS3ClientConfig', () => { From 7cffe1ee2988308870dddee85d9c324eb64f0269 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Tue, 3 Sep 2024 16:06:24 +0200 Subject: [PATCH 17/20] Revert "refactor: move s3 config handling to shared module" This reverts commit ec1ac055cdd9c2e5deb264a9c691c84fd54e9625. --- cdn-server/package.json | 1 - cdn-server/src/s3.ts | 2 +- cdn-server/src/utils.ts | 58 +++++++++++++++++++ controlplane/src/core/build-server.ts | 3 +- controlplane/src/core/util.ts | 46 ++++++++++++++- controlplane/src/types/index.ts | 8 +++ .../test/utils.s3storage.test.ts | 2 +- pnpm-lock.yaml | 35 ++++++----- shared/package.json | 1 - shared/src/types/index.ts | 7 --- shared/src/utils/util.ts | 45 -------------- 11 files changed, 136 insertions(+), 72 deletions(-) create mode 100644 cdn-server/src/utils.ts rename {shared => controlplane}/test/utils.s3storage.test.ts (99%) delete mode 100644 shared/src/types/index.ts diff --git a/cdn-server/package.json b/cdn-server/package.json index 3220708906..5dc762812c 100644 --- a/cdn-server/package.json +++ b/cdn-server/package.json @@ -25,7 +25,6 @@ "dependencies": { "@aws-sdk/client-s3": "^3.529.1", "@hono/node-server": "1.11.0", - "@wundergraph/cosmo-shared": "workspace:*", "@wundergraph/cosmo-cdn": "workspace:*", "dotenv": "^16.4.5", "hono": "4.2.7" diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index ba4ae54e97..eff41947a9 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -1,7 +1,7 @@ import { GetObjectCommand, HeadObjectCommand, NoSuchKey, NotFound, S3Client } from '@aws-sdk/client-s3'; -import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared'; import { BlobNotFoundError, BlobObject, BlobStorage } from '@wundergraph/cosmo-cdn'; import { Context } from 'hono'; +import { createS3ClientConfig, extractS3BucketName } from './utils'; /** * Retrieves objects from S3 given an S3Client and a bucket name diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts new file mode 100644 index 0000000000..e0ea06f2a5 --- /dev/null +++ b/cdn-server/src/utils.ts @@ -0,0 +1,58 @@ +import { S3ClientConfig } from '@aws-sdk/client-s3'; + +/** + * controlplane and cdn are using the same code for handling the s3 storage. + * + * see: controlplane/test/utils.s3storage.test.ts for further details + */ + +interface S3StorageOptions { + url: string; + region?: string; + endpoint?: string; + username?: string; + password?: string; +} + +export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { + const url = new URL(opts.url); + const { region, username, password } = opts; + const forcePathStyle = !isVirtualHostStyleUrl(url); + const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); + + const accessKeyId = url.username || username || ''; + const secretAccessKey = url.password || password || ''; + + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); + } + + if (!region) { + throw new Error('Missing region in S3 configuration.'); + } + + return { + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle, + }; +} + +export function extractS3BucketName(s3Url: string) { + const url = new URL(s3Url); + + if (isVirtualHostStyleUrl(url)) { + return url.hostname.split('.')[0]; + } + + // path based style + return url.pathname.slice(1); +} + +export function isVirtualHostStyleUrl(url: URL) { + return url.hostname.split('.').length > 2; +} diff --git a/controlplane/src/core/build-server.ts b/controlplane/src/core/build-server.ts index c932d0cc21..ce2df020d6 100644 --- a/controlplane/src/core/build-server.ts +++ b/controlplane/src/core/build-server.ts @@ -2,7 +2,6 @@ import Fastify, { FastifyBaseLogger } from 'fastify'; import { S3Client } from '@aws-sdk/client-s3'; import { fastifyConnectPlugin } from '@connectrpc/connect-fastify'; import { cors, createContextValues } from '@connectrpc/connect'; -import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared'; import fastifyCors from '@fastify/cors'; import { pino, stdTimeFunctions, LoggerOptions } from 'pino'; import { compressionBrotli, compressionGzip } from '@connectrpc/connect-node'; @@ -38,7 +37,7 @@ import { BillingRepository } from './repositories/BillingRepository.js'; import { BillingService } from './services/BillingService.js'; import { UserRepository } from './repositories/UserRepository.js'; import { AIGraphReadmeQueue, createAIGraphReadmeWorker } from './workers/AIGraphReadmeWorker.js'; -import { fastifyLoggerId } from './util.js'; +import { fastifyLoggerId, createS3ClientConfig, extractS3BucketName } from './util.js'; import { ApiKeyRepository } from './repositories/ApiKeyRepository.js'; import { createDeleteOrganizationWorker, DeleteOrganizationQueue } from './workers/DeleteOrganizationWorker.js'; diff --git a/controlplane/src/core/util.ts b/controlplane/src/core/util.ts index dcfe82b47e..7256f5136a 100644 --- a/controlplane/src/core/util.ts +++ b/controlplane/src/core/util.ts @@ -1,4 +1,5 @@ import { randomFill } from 'node:crypto'; +import { S3ClientConfig } from '@aws-sdk/client-s3'; import { HandlerContext } from '@connectrpc/connect'; import { EnumStatusCode, @@ -13,7 +14,7 @@ import { uid } from 'uid/secure'; import { AxiosError } from 'axios'; import { isNetworkError, isRetryableError } from 'axios-retry'; import { MemberRole, WebsocketSubprotocol } from '../db/models.js'; -import { AuthContext, DateRange, Label, ResponseMessage } from '../types/index.js'; +import { AuthContext, DateRange, Label, ResponseMessage, S3StorageOptions } from '../types/index.js'; import { isAuthenticationError, isAuthorizationError, isPublicError } from './errors/errors.js'; import { GraphKeyAuthContext } from './services/GraphApiTokenAuthenticator.js'; @@ -372,3 +373,46 @@ export function getValueOrDefault(map: Map, key: K, constructor: () export function webhookAxiosRetryCond(err: AxiosError) { return isNetworkError(err) || isRetryableError(err); } + +export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { + const url = new URL(opts.url); + const { region, username, password } = opts; + const forcePathStyle = !isVirtualHostStyleUrl(url); + const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); + + const accessKeyId = url.username || username || ''; + const secretAccessKey = url.password || password || ''; + + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); + } + + if (!region) { + throw new Error('Missing region in S3 configuration.'); + } + + return { + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle, + }; +} + +export function extractS3BucketName(s3Url: string) { + const url = new URL(s3Url); + + if (isVirtualHostStyleUrl(url)) { + return url.hostname.split('.')[0]; + } + + // path based style + return url.pathname.slice(1); +} + +export function isVirtualHostStyleUrl(url: URL) { + return url.hostname.split('.').length > 2; +} diff --git a/controlplane/src/types/index.ts b/controlplane/src/types/index.ts index 946e49e49a..08aeacfc58 100644 --- a/controlplane/src/types/index.ts +++ b/controlplane/src/types/index.ts @@ -576,3 +576,11 @@ export interface SchemaLintIssues { warnings: LintIssueResult[]; errors: LintIssueResult[]; } + +export interface S3StorageOptions { + url: string; + region?: string; + endpoint?: string; + username?: string; + password?: string; +} diff --git a/shared/test/utils.s3storage.test.ts b/controlplane/test/utils.s3storage.test.ts similarity index 99% rename from shared/test/utils.s3storage.test.ts rename to controlplane/test/utils.s3storage.test.ts index d197619297..7919e2a3bd 100644 --- a/shared/test/utils.s3storage.test.ts +++ b/controlplane/test/utils.s3storage.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src'; +import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src/core/util.js'; describe('S3 Utils', () => { describe('createS3ClientConfig', () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d3beaf1f5..ad0f0a8e71 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,9 +104,6 @@ importers: '@wundergraph/cosmo-cdn': specifier: workspace:* version: link:cdn - '@wundergraph/cosmo-shared': - specifier: workspace:* - version: link:../shared dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -706,9 +703,6 @@ importers: shared: dependencies: - '@aws-sdk/client-s3': - specifier: ^3.529.1 - version: 3.529.1 '@bufbuild/protobuf': specifier: ^1.9.0 version: 1.9.0 @@ -1113,7 +1107,7 @@ packages: resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==} dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.523.0 + '@aws-sdk/types': 3.433.0 tslib: 1.14.1 dev: false @@ -1121,7 +1115,7 @@ packages: resolution: {integrity: sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==} dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.523.0 + '@aws-sdk/types': 3.433.0 tslib: 1.14.1 dev: false @@ -1173,7 +1167,7 @@ packages: /@aws-crypto/util@3.0.0: resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==} dependencies: - '@aws-sdk/types': 3.523.0 + '@aws-sdk/types': 3.433.0 '@aws-sdk/util-utf8-browser': 3.259.0 tslib: 1.14.1 dev: false @@ -1669,6 +1663,14 @@ packages: - aws-crt dev: false + /@aws-sdk/types@3.433.0: + resolution: {integrity: sha512-0jEE2mSrNDd8VGFjTc1otYrwYPIkzZJEIK90ZxisKvQ/EURGBhNzWn7ejWB9XCMFT6XumYLBR0V9qq5UPisWtA==} + engines: {node: '>=14.0.0'} + dependencies: + '@smithy/types': 2.4.0 + tslib: 2.6.2 + dev: false + /@aws-sdk/types@3.523.0: resolution: {integrity: sha512-AqGIu4u+SxPiUuNBp2acCVcq80KDUFjxe6e3cMTvKWTzCbrVk1AXv0dAaJnCmdkWIha6zJDWxpIk/aL4EGhZ9A==} engines: {node: '>=14.0.0'} @@ -11542,6 +11544,13 @@ packages: tslib: 2.6.2 dev: false + /@smithy/types@2.4.0: + resolution: {integrity: sha512-iH1Xz68FWlmBJ9vvYeHifVMWJf82ONx+OybPW8ZGf5wnEv2S0UXcU4zwlwJkRXuLKpcSLHrraHbn2ucdVXLb4g==} + engines: {node: '>=14.0.0'} + dependencies: + tslib: 2.6.2 + dev: false + /@smithy/url-parser@2.1.3: resolution: {integrity: sha512-X1NRA4WzK/ihgyzTpeGvI9Wn45y8HmqF4AZ/FazwAv8V203Ex+4lXqcYI70naX9ETqbqKVzFk88W6WJJzCggTQ==} dependencies: @@ -12683,7 +12692,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/type-utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) '@typescript-eslint/utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) - debug: 4.3.4 + debug: 4.3.5 eslint: 8.52.0 graphemer: 1.4.0 ignore: 5.2.4 @@ -12784,7 +12793,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.5.2) - debug: 4.3.4 + debug: 4.3.5 eslint: 8.52.0 typescript: 5.5.2 transitivePeerDependencies: @@ -12851,7 +12860,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.5.2) '@typescript-eslint/utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) - debug: 4.3.4 + debug: 4.3.5 eslint: 8.52.0 tsutils: 3.21.0(typescript@5.5.2) typescript: 5.5.2 @@ -16197,7 +16206,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.4 + debug: 4.3.5 enhanced-resolve: 5.15.0 eslint: 8.52.0 eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0)(eslint-import-resolver-typescript@3.5.5)(eslint@8.52.0) diff --git a/shared/package.json b/shared/package.json index b78f2b4680..bdf24028af 100644 --- a/shared/package.json +++ b/shared/package.json @@ -33,7 +33,6 @@ "license": "Apache-2.0", "dependencies": { "@bufbuild/protobuf": "^1.9.0", - "@aws-sdk/client-s3": "^3.529.1", "@graphql-tools/schema": "^8.5.1", "@graphql-tools/utils": "^9.2.1", "@wundergraph/composition": "workspace:*", diff --git a/shared/src/types/index.ts b/shared/src/types/index.ts deleted file mode 100644 index bce07f22fe..0000000000 --- a/shared/src/types/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export interface S3StorageOptions { - url: string; - region?: string; - endpoint?: string; - username?: string; - password?: string; -} diff --git a/shared/src/utils/util.ts b/shared/src/utils/util.ts index f22a701141..9acb0f9e3e 100644 --- a/shared/src/utils/util.ts +++ b/shared/src/utils/util.ts @@ -1,6 +1,4 @@ -import * as S3 from '@aws-sdk/client-s3'; import { SubscriptionProtocol, WebsocketSubprotocol } from '../router-config/builder.js'; -import { S3StorageOptions } from '../types/index.js'; export function delay(t: number) { return new Promise((resolve) => setTimeout(resolve, t)); @@ -78,46 +76,3 @@ export function isValidWebsocketSubprotocol(protocol: WebsocketSubprotocol) { } } } - -export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3.S3ClientConfig { - const url = new URL(opts.url); - const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); - const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - - const accessKeyId = url.username || username || ''; - const secretAccessKey = url.password || password || ''; - - if (!accessKeyId || !secretAccessKey) { - throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); - } - - if (!region) { - throw new Error('Missing region in S3 configuration.'); - } - - return { - region, - endpoint, - credentials: { - accessKeyId, - secretAccessKey, - }, - forcePathStyle, - }; -} - -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); - - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; - } - - // path based style - return url.pathname.slice(1); -} - -export function isVirtualHostStyleUrl(url: URL) { - return url.hostname.split('.').length > 2; -} From cfe1c8c639befdc145dd4a711fb92458d707e3e4 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Wed, 4 Sep 2024 22:34:26 +0200 Subject: [PATCH 18/20] fix: not respecting nested virtual hosts with path --- cdn-server/src/s3.ts | 10 +- cdn-server/src/utils.ts | 14 +- controlplane/src/core/build-server.ts | 3 +- controlplane/src/core/env.schema.ts | 17 +- controlplane/src/core/util.ts | 13 +- controlplane/src/index.ts | 2 + controlplane/src/types/index.ts | 1 + controlplane/test/utils.s3storage.test.ts | 179 ++++++++++-------- docker-compose.full.yml | 2 + .../charts/cdn/templates/config-map.yaml | 3 + .../charts/cdn/templates/deployment.yaml | 8 +- helm/cosmo/charts/cdn/values.yaml | 2 + .../controlplane/templates/config-map.yaml | 3 + .../controlplane/templates/deployment.yaml | 7 + helm/cosmo/charts/controlplane/values.yaml | 2 + helm/cosmo/values.yaml | 4 + 16 files changed, 176 insertions(+), 94 deletions(-) diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index eff41947a9..8c3db88334 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -91,15 +91,19 @@ export const createS3BlobStorage = (storageUrl: string): BlobStorage => { const endpoint = url.searchParams.get('endpoint') ?? process.env.S3_ENDPOINT; const username = process.env.S3_ACCESS_KEY_ID || ''; const password = process.env.S3_SECRET_ACCESS_KEY || ''; + const forcePathStyle = process.env.S3_FORCE_PATH_STYLE === 'true'; - const bucketName = extractS3BucketName(storageUrl); - const s3Config = createS3ClientConfig(bucketName, { + const opts = { url: storageUrl, region, endpoint, username, password, - }); + forcePathStyle, + }; + + const bucketName = extractS3BucketName(opts); + const s3Config = createS3ClientConfig(bucketName, opts); const s3Client = new S3Client(s3Config); return new S3BlobStorage(s3Client, bucketName); diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts index e0ea06f2a5..81af3b2c5d 100644 --- a/cdn-server/src/utils.ts +++ b/cdn-server/src/utils.ts @@ -12,12 +12,13 @@ interface S3StorageOptions { endpoint?: string; username?: string; password?: string; + forcePathStyle?: boolean; } export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { const url = new URL(opts.url); const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); + const forcePathStyle = opts.forcePathStyle ?? !isVirtualHostStyleUrl(url); const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); const accessKeyId = url.username || username || ''; @@ -42,15 +43,14 @@ export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions) }; } -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); +export function extractS3BucketName(opts: S3StorageOptions) { + const url = new URL(opts.url); - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; + if (opts.forcePathStyle || !isVirtualHostStyleUrl(url)) { + return url.pathname.slice(1); } - // path based style - return url.pathname.slice(1); + return url.hostname.split('.')[0]; } export function isVirtualHostStyleUrl(url: URL) { diff --git a/controlplane/src/core/build-server.ts b/controlplane/src/core/build-server.ts index ce2df020d6..70920a5f39 100644 --- a/controlplane/src/core/build-server.ts +++ b/controlplane/src/core/build-server.ts @@ -92,6 +92,7 @@ export interface BuildConfig { region?: string; username?: string; password?: string; + forcePathStyle?: boolean; }; mailer: { smtpEnabled: boolean; @@ -353,7 +354,7 @@ export default async function build(opts: BuildConfig) { throw new Error('S3 storage URL is required'); } - const bucketName = extractS3BucketName(opts.s3Storage.url); + const bucketName = extractS3BucketName(opts.s3Storage); const s3Config = createS3ClientConfig(bucketName, opts.s3Storage); const s3Client = new S3Client(s3Config); diff --git a/controlplane/src/core/env.schema.ts b/controlplane/src/core/env.schema.ts index c59c05294b..f44ba0f719 100644 --- a/controlplane/src/core/env.schema.ts +++ b/controlplane/src/core/env.schema.ts @@ -126,7 +126,22 @@ export const envVariables = z */ S3_ACCESS_KEY_ID: z.string().optional(), S3_SECRET_ACCESS_KEY: z.string().optional(), - + /** + * Enforces path style URLs handling, e.g.: + * https://username:password@virtualhost.r2.cloudflarestorage.com/cosmo-cdn + * + * S3_STORAGE_URL="https://username:password@virtualhost.r2.cloudflarestorage.com/cosmo-cdn" + * + * S3_REGION="auto" + * S3_ENDPOINT="https://virtualhost.r2.cloudflarestorage.com" + * S3_FORCE_PATH_STYLE="true" + * + * The bucket will be "cosmo-cdn" otherwise it would be "virtualhost" + */ + S3_FORCE_PATH_STYLE: z + .string() + .transform((val) => val === 'true') + .default('true'), /** * Email */ diff --git a/controlplane/src/core/util.ts b/controlplane/src/core/util.ts index 7256f5136a..12332772f7 100644 --- a/controlplane/src/core/util.ts +++ b/controlplane/src/core/util.ts @@ -377,7 +377,7 @@ export function webhookAxiosRetryCond(err: AxiosError) { export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { const url = new URL(opts.url); const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); + const forcePathStyle = opts.forcePathStyle ?? !isVirtualHostStyleUrl(url); const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); const accessKeyId = url.username || username || ''; @@ -402,15 +402,14 @@ export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions) }; } -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); +export function extractS3BucketName(opts: S3StorageOptions) { + const url = new URL(opts.url); - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; + if (opts.forcePathStyle || !isVirtualHostStyleUrl(url)) { + return url.pathname.slice(1); } - // path based style - return url.pathname.slice(1); + return url.hostname.split('.')[0]; } export function isVirtualHostStyleUrl(url: URL) { diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index d7aba46ab4..ef3f008977 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -45,6 +45,7 @@ const { S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, + S3_FORCE_PATH_STYLE, SMTP_ENABLED, SMTP_HOST, SMTP_PORT, @@ -126,6 +127,7 @@ const options: BuildConfig = { endpoint: S3_ENDPOINT, username: S3_ACCESS_KEY_ID, password: S3_SECRET_ACCESS_KEY, + forcePathStyle: S3_FORCE_PATH_STYLE, }, mailer: { smtpEnabled: SMTP_ENABLED, diff --git a/controlplane/src/types/index.ts b/controlplane/src/types/index.ts index 08aeacfc58..8c04f8f66e 100644 --- a/controlplane/src/types/index.ts +++ b/controlplane/src/types/index.ts @@ -583,4 +583,5 @@ export interface S3StorageOptions { endpoint?: string; username?: string; password?: string; + forcePathStyle?: boolean; } diff --git a/controlplane/test/utils.s3storage.test.ts b/controlplane/test/utils.s3storage.test.ts index 7919e2a3bd..772480fafe 100644 --- a/controlplane/test/utils.s3storage.test.ts +++ b/controlplane/test/utils.s3storage.test.ts @@ -2,177 +2,208 @@ import { describe, expect, test } from 'vitest'; import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src/core/util.js'; describe('S3 Utils', () => { - describe('createS3ClientConfig', () => { - test('correctly configures an S3 client for a path-style URL', () => { - const bucketName = 'cosmo'; + describe('createS3ClientConfig with forced path style', () => { + test('that it correctly configures an S3 client for a path-style URL', () => { const opts = { - url: 'http://username:password@minio:9000/cosmo', - region: 'auto', - endpoint: undefined, - username: '', - password: '', + url: 'https://cosmo-controlplane-bucket.provider.com/cosmo', + region: 'us-east-1', + endpoint: '', + username: 'testUser', + password: 'testPass', + forcePathStyle: true, }; + const bucketName = extractS3BucketName(opts); const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ - region: 'auto', - endpoint: 'http://minio:9000', + region: 'us-east-1', + endpoint: 'https://cosmo-controlplane-bucket.provider.com', credentials: { - accessKeyId: 'username', - secretAccessKey: 'password', + accessKeyId: 'testUser', + secretAccessKey: 'testPass', }, forcePathStyle: true, }); + expect(bucketName).toBe('cosmo'); }); - test('correctly configures an S3 client for a virtual-hosted-style URL with provided endpoint', () => { - const bucketName = 'cosmo-controlplane-bucket'; + test('that it correctly configures an S3 client for a path-style URL when an endpoint is set', () => { const opts = { - url: 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com', + url: 'https://cosmo-controlplane-bucket.provider.com/cosmo', region: 'us-east-1', - endpoint: 's3.amazonaws.com', - username: '', - password: '', + endpoint: 'custom-endpoint.com', + username: 'testUser', + password: 'testPass', + forcePathStyle: true, }; + const bucketName = extractS3BucketName(opts); const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'us-east-1', - endpoint: 's3.amazonaws.com', + endpoint: 'custom-endpoint.com', credentials: { - accessKeyId: 'username', - secretAccessKey: 'password', + accessKeyId: 'testUser', + secretAccessKey: 'testPass', }, - forcePathStyle: false, + forcePathStyle: true, }); + expect(bucketName).toBe('cosmo'); }); - test('correctly configures an S3 client for a virtual-hosted-style URL without provided endpoint', () => { - const bucketName = 'cosmo-controlplane-bucket'; + test('that it correctly configures an S3 client for a path-style URL which also has virtual hosts', () => { const opts = { - url: 'https://username:password@cosmo-controlplane-bucket.s3.amazonaws.com', + url: 'https://username:password@xxxxxxxxxxxxxxxxxxx.r2.cloudflarestorage.com/cosmo-cdn', region: 'us-east-1', - endpoint: undefined, - username: '', - password: '', + endpoint: '', + forcePathStyle: true, }; + const bucketName = extractS3BucketName(opts); const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'us-east-1', - endpoint: 'https://s3.amazonaws.com', + endpoint: 'https://xxxxxxxxxxxxxxxxxxx.r2.cloudflarestorage.com', credentials: { accessKeyId: 'username', secretAccessKey: 'password', }, - forcePathStyle: false, + forcePathStyle: true, }); + expect(bucketName).toBe('cosmo-cdn'); }); + }); - test('throws an AuthenticationError if credentials are missing', () => { - const bucketName = 'cosmo-controlplane-bucket'; + // Section: Virtual-hosted-style URL tests without forced path style + describe('createS3ClientConfig without forced path style', () => { + test('that it correctly configures an S3 client for a virtual-hosted-style URL', () => { const opts = { - url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com', + url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com/cosmo', region: 'us-east-1', endpoint: '', - username: '', - password: '', + username: 'testUser', + password: 'testPass', + forcePathStyle: false, }; - expect(() => createS3ClientConfig(bucketName, opts)).toThrowError( - 'Missing S3 credentials. Please provide access key ID and secret access key.', - ); + const bucketName = extractS3BucketName(opts); + const config = createS3ClientConfig(bucketName, opts); + + expect(config).toEqual({ + region: 'us-east-1', + endpoint: 'https://s3.amazonaws.com', + credentials: { + accessKeyId: 'testUser', + secretAccessKey: 'testPass', + }, + forcePathStyle: false, + }); + expect(bucketName).toBe('cosmo-controlplane-bucket'); }); - test('correctly configures an S3 client when credentials are provided in opts', () => { - const bucketName = 'cosmo-controlplane-bucket'; + test('that it correctly configures an S3 client for a virtual-hosted-style URL with a custom endpoint', () => { const opts = { - url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com', + url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com/cosmo', region: 'us-east-1', - endpoint: '', + endpoint: 's3.amazonaws.com', username: 'testUser', password: 'testPass', + forcePathStyle: false, }; + const bucketName = extractS3BucketName(opts); const config = createS3ClientConfig(bucketName, opts); expect(config).toEqual({ region: 'us-east-1', - endpoint: 'https://s3.amazonaws.com', + endpoint: 's3.amazonaws.com', credentials: { accessKeyId: 'testUser', secretAccessKey: 'testPass', }, forcePathStyle: false, }); + expect(bucketName).toBe('cosmo-controlplane-bucket'); }); - test('throws an error if region is missing', () => { - const bucketName = 'cosmo-controlplane-bucket'; + test('that it correctly configures an S3 client with multiple subdomains path styled', () => { const opts = { - url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com', - region: '', + url: 'https://minio.east.domain.com/mybucket', + region: 'auto', endpoint: '', username: 'testUser', password: 'testPass', + forcePathStyle: true, }; - expect(() => createS3ClientConfig(bucketName, opts)).toThrowError('Missing region in S3 configuration.'); + const bucketName = extractS3BucketName(opts); + const config = createS3ClientConfig(bucketName, opts); + + expect(config).toEqual({ + region: 'auto', + endpoint: 'https://minio.east.domain.com', + credentials: { + accessKeyId: 'testUser', + secretAccessKey: 'testPass', + }, + forcePathStyle: true, + }); + expect(bucketName).toBe('mybucket'); }); }); - describe('extractS3BucketName', () => { - test('returns the correct bucket name for a virtual-hosted-style URL', () => { - const s3Url = 'https://cosmo-controlplane-bucket.s3.amazonaws.com/some/object'; + describe('isVirtualHostStyleUrl tests', () => { + test('that it returns true for a virtual-hosted-style URL', () => { + const url = new URL('https://cosmo-controlplane-bucket.s3.amazonaws.com'); - const bucketName = extractS3BucketName(s3Url); + const result = isVirtualHostStyleUrl(url); - expect(bucketName).toBe('cosmo-controlplane-bucket'); + expect(result).toBe(true); }); - test('returns the correct bucket name for a path-style URL', () => { - const s3Url = 'http://minio:9000/cosmo'; + test('that it returns false for a path-style URL', () => { + const url = new URL('http://minio:9000/cosmo'); - const bucketName = extractS3BucketName(s3Url); + const result = isVirtualHostStyleUrl(url); - expect(bucketName).toBe('cosmo'); + expect(result).toBe(false); }); - test('returns the correct bucket name when the URL has multiple path segments', () => { - const s3Url = 'http://username:password@localhost:9000/foo'; + test('that it returns false for a custom domain without bucket name in the hostname', () => { + const url = new URL('https://example.com/cosmo'); - const bucketName = extractS3BucketName(s3Url); + const result = isVirtualHostStyleUrl(url); - expect(bucketName).toBe('foo'); + expect(result).toBe(false); }); }); - describe('isVirtualHostStyleUrl', () => { - test('returns true for a virtual-hosted-style URL', () => { - const url = new URL('https://cosmo-controlplane-bucket.s3.amazonaws.com'); + describe('extractS3BucketName tests', () => { + test('that it returns the correct bucket name for a virtual-hosted-style URL', () => { + const opts = { url: 'https://cosmo-controlplane-bucket.s3.amazonaws.com/some/object' }; - const result = isVirtualHostStyleUrl(url); + const bucketName = extractS3BucketName(opts); - expect(result).toBe(true); + expect(bucketName).toBe('cosmo-controlplane-bucket'); }); - test('returns false for a path-style URL', () => { - const url = new URL('http://minio:9000/cosmo'); + test('that it returns the correct bucket name for a path-style URL', () => { + const opts = { url: 'http://minio:9000/cosmo' }; - const result = isVirtualHostStyleUrl(url); + const bucketName = extractS3BucketName(opts); - expect(result).toBe(false); + expect(bucketName).toBe('cosmo'); }); - test('returns false for a custom domain without bucket name in the hostname', () => { - const url = new URL('https://example.com/cosmo'); + test('that it returns the correct bucket name when the URL has multiple path segments', () => { + const opts = { url: 'http://username:password@localhost:9000/foo' }; - const result = isVirtualHostStyleUrl(url); + const bucketName = extractS3BucketName(opts); - expect(result).toBe(false); + expect(bucketName).toBe('foo'); }); }); }); diff --git a/docker-compose.full.yml b/docker-compose.full.yml index 4ab2fb913c..0378112c77 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -213,6 +213,7 @@ services: S3_REGION: ${S3_REGION_CDN:-${S3_REGION:-auto}} S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE:-true} ports: - '11000:11000' networks: @@ -278,6 +279,7 @@ services: S3_REGION: ${S3_REGION_CONTROLPLANE:-${S3_REGION:-auto}} S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE:-true} CDN_BASE_URL: 'http://cdn:11000' REDIS_HOST: redis REDIS_PORT: 6379 diff --git a/helm/cosmo/charts/cdn/templates/config-map.yaml b/helm/cosmo/charts/cdn/templates/config-map.yaml index ce0c806ab6..5c6a51f885 100644 --- a/helm/cosmo/charts/cdn/templates/config-map.yaml +++ b/helm/cosmo/charts/cdn/templates/config-map.yaml @@ -14,3 +14,6 @@ data: {{- if .Values.configuration.s3Endpoint }} s3Endpoint: "{{ .Values.configuration.s3Endpoint }}" {{- end }} + {{- if .Values.configuration.s3ForcePathStyle }} + s3ForcePathStyle: "{{ .Values.configuration.s3ForcePathStyle }}" + {{ end }} diff --git a/helm/cosmo/charts/cdn/templates/deployment.yaml b/helm/cosmo/charts/cdn/templates/deployment.yaml index 3edfdbf935..a0460f9e3d 100644 --- a/helm/cosmo/charts/cdn/templates/deployment.yaml +++ b/helm/cosmo/charts/cdn/templates/deployment.yaml @@ -64,7 +64,13 @@ spec: configMapKeyRef: name: {{ include "cdn.fullname" . }}-configmap key: s3Region - + {{- if .Values.configuration.s3ForcePathStyle }} + - name: S3_FORCE_PATH_STYLE + valueFrom: + configMapKeyRef: + name: {{ include "cdn.fullname" . }}-configmap + key: s3ForcePathStyle + {{- end }} {{- if .Values.configuration.s3Endpoint }} - name: S3_ENDPOINT valueFrom: diff --git a/helm/cosmo/charts/cdn/values.yaml b/helm/cosmo/charts/cdn/values.yaml index 4d7cb54f1f..56cfa90a6c 100644 --- a/helm/cosmo/charts/cdn/values.yaml +++ b/helm/cosmo/charts/cdn/values.yaml @@ -152,6 +152,8 @@ configuration: s3AccessKeyId: '' # -- s3 secret access key, can be used instead of [username]:[password] in the url s3SecretAccessKey: '' + # -- Forces usage of path style urls for S3. Default is true. + s3ForcePathStyle: 'true' # -- Existing secret in the same namespace containing the authJwtSecret and s3StorageUrl. The secret keys have to match with current secret. existingSecret: "" diff --git a/helm/cosmo/charts/controlplane/templates/config-map.yaml b/helm/cosmo/charts/controlplane/templates/config-map.yaml index 2fe430080e..188b9d1c3d 100644 --- a/helm/cosmo/charts/controlplane/templates/config-map.yaml +++ b/helm/cosmo/charts/controlplane/templates/config-map.yaml @@ -44,4 +44,7 @@ data: s3Region: "{{ .Values.configuration.s3Region }}" {{- if .Values.configuration.s3Endpoint }} s3Endpoint: "{{ .Values.configuration.s3Endpoint }}" + {{ end }} + {{- if .Values.configuration.s3ForcePathStyle }} + s3ForcePathStyle: "{{ .Values.configuration.s3ForcePathStyle }}" {{ end }} \ No newline at end of file diff --git a/helm/cosmo/charts/controlplane/templates/deployment.yaml b/helm/cosmo/charts/controlplane/templates/deployment.yaml index 22364231aa..a79825c99f 100644 --- a/helm/cosmo/charts/controlplane/templates/deployment.yaml +++ b/helm/cosmo/charts/controlplane/templates/deployment.yaml @@ -238,6 +238,13 @@ spec: name: {{ include "controlplane.fullname" . }}-configmap key: s3Endpoint {{- end }} + {{- if .Values.configuration.s3ForcePathStyle }} + - name: S3_FORCE_PATH_STYLE + valueFrom: + configMapKeyRef: + name: {{ include "controlplane.fullname" . }}-configmap + key: s3ForcePathStyle + {{- end }} {{- if .Values.configuration.s3AccessKeyId }} - name: S3_ACCESS_KEY_ID valueFrom: diff --git a/helm/cosmo/charts/controlplane/values.yaml b/helm/cosmo/charts/controlplane/values.yaml index 5a8645af9b..274ca16684 100644 --- a/helm/cosmo/charts/controlplane/values.yaml +++ b/helm/cosmo/charts/controlplane/values.yaml @@ -192,6 +192,8 @@ configuration: s3AccessKeyId: '' # -- s3 secret access key, can be used instead of [username]:[password] in the url s3SecretAccessKey: '' + # -- Forces usage of path style urls for S3. Default is true. + s3ForcePathStyle: 'true' stripeSecretKey: '' stripeWebhookSecret: '' # -- The default billing plan, eg `developer@1` diff --git a/helm/cosmo/values.yaml b/helm/cosmo/values.yaml index d9ed8b3f36..1acaadd8f6 100644 --- a/helm/cosmo/values.yaml +++ b/helm/cosmo/values.yaml @@ -128,6 +128,8 @@ cdn: s3AccessKeyId: '' # -- s3 secret access key, can be used instead of [username]:[password] in the url s3SecretAccessKey: '' + # -- Forces usage of path style urls for S3. Default is true. + s3ForcePathStyle: 'true' # Cosmo Controlplane. For more options, please refer to the README.md controlplane: @@ -202,6 +204,8 @@ controlplane: s3AccessKeyId: '' # -- s3 secret access key, can be used instead of [username]:[password] in the url s3SecretAccessKey: '' + # -- Forces usage of path style urls for S3. Default is true. + s3ForcePathStyle: 'true' cdnBaseUrl: 'http://cosmo-cdn:8787' From e01f183f0b256d7501a9d0ec11e122fd1bd01af1 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Wed, 4 Sep 2024 22:45:23 +0200 Subject: [PATCH 19/20] chore: wire through new attribute --- helm/cosmo/README.md | 2 ++ helm/cosmo/charts/cdn/README.md | 1 + helm/cosmo/charts/controlplane/README.md | 1 + 3 files changed, 4 insertions(+) diff --git a/helm/cosmo/README.md b/helm/cosmo/README.md index ae39faf638..e6ba36d65b 100644 --- a/helm/cosmo/README.md +++ b/helm/cosmo/README.md @@ -37,6 +37,7 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | cdn.commonLabels | object | `{}` | Add labels to all deployed resources | | cdn.configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | cdn.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| cdn.configuration.s3ForcePathStyle | string | `"true"` | Forces usage of path style urls for S3. Default is true. | | cdn.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | cdn.configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | cdn.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | @@ -70,6 +71,7 @@ This is the official Helm Chart for WunderGraph Cosmo - The Full Lifecycle Graph | controlplane.configuration.redisPort | int | `6379` | | | controlplane.configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | controlplane.configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| controlplane.configuration.s3ForcePathStyle | string | `"true"` | Forces usage of path style urls for S3. Default is true. | | controlplane.configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | controlplane.configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | controlplane.configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | diff --git a/helm/cosmo/charts/cdn/README.md b/helm/cosmo/charts/cdn/README.md index b91287fba5..7db329cbc2 100644 --- a/helm/cosmo/charts/cdn/README.md +++ b/helm/cosmo/charts/cdn/README.md @@ -18,6 +18,7 @@ WunderGraph Cosmo CDN | commonLabels | object | `{}` | Add labels to all deployed resources | | configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| configuration.s3ForcePathStyle | string | `"true"` | Forces usage of path style urls for S3. Default is true. | | configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | deploymentStrategy | object | `{}` | | diff --git a/helm/cosmo/charts/controlplane/README.md b/helm/cosmo/charts/controlplane/README.md index 2cf9efb887..d2a6f748df 100644 --- a/helm/cosmo/charts/controlplane/README.md +++ b/helm/cosmo/charts/controlplane/README.md @@ -51,6 +51,7 @@ WunderGraph Cosmo Controlplane | configuration.redisTlsKey | string | `""` | | | configuration.s3AccessKeyId | string | `""` | s3 access key id, can be used instead of [username]:[password] in the url | | configuration.s3Endpoint | string | `""` | The endpoint of the S3 bucket. | +| configuration.s3ForcePathStyle | string | `"true"` | Forces usage of path style urls for S3. Default is true. | | configuration.s3Region | string | `"auto"` | The region where the S3 bucket is located. | | configuration.s3SecretAccessKey | string | `""` | s3 secret access key, can be used instead of [username]:[password] in the url | | configuration.s3StorageUrl | string | `"http://minio:changeme@cosmo-minio:9000/cosmo"` | | From 9bf9a1b83fb2a1e996b49706c6c3f29a636ca0a3 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Thu, 5 Sep 2024 11:07:52 +0200 Subject: [PATCH 20/20] chore: remove comment --- controlplane/test/utils.s3storage.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/controlplane/test/utils.s3storage.test.ts b/controlplane/test/utils.s3storage.test.ts index 772480fafe..fb98e26e98 100644 --- a/controlplane/test/utils.s3storage.test.ts +++ b/controlplane/test/utils.s3storage.test.ts @@ -77,7 +77,6 @@ describe('S3 Utils', () => { }); }); - // Section: Virtual-hosted-style URL tests without forced path style describe('createS3ClientConfig without forced path style', () => { test('that it correctly configures an S3 client for a virtual-hosted-style URL', () => { const opts = {