Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: cdn and controlplane not respecting nested subdomains #1145

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6a4face
feat(controlplane): enable using aws based s3 storage
AndreasZeissner Sep 1, 2024
ac0e507
feat(cdn-server): enable aws s3 storage backend
AndreasZeissner Sep 1, 2024
ebb29b5
feat(cdn): enable s3 region and endpoint handling
AndreasZeissner Sep 1, 2024
6a5c6f6
feat(controlplane): enable s3 region and endpoint handling
AndreasZeissner Sep 1, 2024
85daa7b
feat: enable s3region and s3Endpoint for cdn and controlplane
AndreasZeissner Sep 1, 2024
2e8cce4
chore: update docs
AndreasZeissner Sep 1, 2024
f8aab09
chore: enable S3 handling in compose files
AndreasZeissner Sep 1, 2024
7bf5dbf
chore: remove unecessary defaulting
AndreasZeissner Sep 2, 2024
9cac8fe
feat: enable using aws credential handling for s3
AndreasZeissner Sep 2, 2024
1287a5b
refactor: do not use aws namings to avoid shadowings
AndreasZeissner Sep 2, 2024
8385160
chore: use right env var for controlplane region
AndreasZeissner Sep 2, 2024
d6a5228
chore: add backwards compatible env vars
AndreasZeissner Sep 2, 2024
14c321f
docs: update comment
AndreasZeissner Sep 2, 2024
0b382ff
docs: update comment
AndreasZeissner Sep 2, 2024
5c6d6a6
chore: add username and password handling
AndreasZeissner Sep 2, 2024
ec1ac05
refactor: move s3 config handling to shared module
AndreasZeissner Sep 3, 2024
7cffe1e
Revert "refactor: move s3 config handling to shared module"
AndreasZeissner Sep 3, 2024
89b1516
Merge branch 'main' of github.com:wundergraph/cosmo into andi/eng-560…
AndreasZeissner Sep 4, 2024
cfe1c8c
fix: not respecting nested virtual hosts with path
AndreasZeissner Sep 4, 2024
e01f183
chore: wire through new attribute
AndreasZeissner Sep 4, 2024
3841ee3
Merge branch 'main' into andi/eng-5603-make-controlplane-compatible-w…
AndreasZeissner Sep 5, 2024
9bf9a1b
chore: remove comment
AndreasZeissner Sep 5, 2024
cdb15b3
Merge branch 'main' into andi/eng-5603-make-controlplane-compatible-w…
AndreasZeissner Sep 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions cdn-server/src/s3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,19 @@ export const createS3BlobStorage = (storageUrl: string): BlobStorage => {
const endpoint = url.searchParams.get('endpoint') ?? process.env.S3_ENDPOINT;
const username = process.env.S3_ACCESS_KEY_ID || '';
const password = process.env.S3_SECRET_ACCESS_KEY || '';
const forcePathStyle = process.env.S3_FORCE_PATH_STYLE === 'true';

const bucketName = extractS3BucketName(storageUrl);
const s3Config = createS3ClientConfig(bucketName, {
const opts = {
url: storageUrl,
region,
endpoint,
username,
password,
});
forcePathStyle,
};

const bucketName = extractS3BucketName(opts);
const s3Config = createS3ClientConfig(bucketName, opts);
const s3Client = new S3Client(s3Config);

return new S3BlobStorage(s3Client, bucketName);
Expand Down
14 changes: 7 additions & 7 deletions cdn-server/src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ interface S3StorageOptions {
endpoint?: string;
username?: string;
password?: string;
forcePathStyle?: boolean;
}

export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig {
const url = new URL(opts.url);
const { region, username, password } = opts;
const forcePathStyle = !isVirtualHostStyleUrl(url);
const forcePathStyle = opts.forcePathStyle ?? !isVirtualHostStyleUrl(url);
const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, ''));

const accessKeyId = url.username || username || '';
Expand All @@ -42,15 +43,14 @@ export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions)
};
}

export function extractS3BucketName(s3Url: string) {
const url = new URL(s3Url);
export function extractS3BucketName(opts: S3StorageOptions) {
const url = new URL(opts.url);

if (isVirtualHostStyleUrl(url)) {
return url.hostname.split('.')[0];
if (opts.forcePathStyle || !isVirtualHostStyleUrl(url)) {
return url.pathname.slice(1);
}

// path based style
return url.pathname.slice(1);
return url.hostname.split('.')[0];
}

export function isVirtualHostStyleUrl(url: URL) {
Expand Down
3 changes: 2 additions & 1 deletion controlplane/src/core/build-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ export interface BuildConfig {
region?: string;
username?: string;
password?: string;
forcePathStyle?: boolean;
};
mailer: {
smtpEnabled: boolean;
Expand Down Expand Up @@ -353,7 +354,7 @@ export default async function build(opts: BuildConfig) {
throw new Error('S3 storage URL is required');
}

const bucketName = extractS3BucketName(opts.s3Storage.url);
const bucketName = extractS3BucketName(opts.s3Storage);
const s3Config = createS3ClientConfig(bucketName, opts.s3Storage);

const s3Client = new S3Client(s3Config);
Expand Down
17 changes: 16 additions & 1 deletion controlplane/src/core/env.schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,22 @@ export const envVariables = z
*/
S3_ACCESS_KEY_ID: z.string().optional(),
S3_SECRET_ACCESS_KEY: z.string().optional(),

/**
* Enforces path style URLs handling, e.g.:
* https://username:[email protected]/cosmo-cdn
*
* S3_STORAGE_URL="https://username:[email protected]/cosmo-cdn"
*
* S3_REGION="auto"
* S3_ENDPOINT="https://virtualhost.r2.cloudflarestorage.com"
* S3_FORCE_PATH_STYLE="true"
*
* The bucket will be "cosmo-cdn" otherwise it would be "virtualhost"
*/
S3_FORCE_PATH_STYLE: z
.string()
.transform((val) => val === 'true')
.default('true'),
/**
* Email
*/
Expand Down
13 changes: 6 additions & 7 deletions controlplane/src/core/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ export function webhookAxiosRetryCond(err: AxiosError) {
export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig {
const url = new URL(opts.url);
const { region, username, password } = opts;
const forcePathStyle = !isVirtualHostStyleUrl(url);
const forcePathStyle = opts.forcePathStyle ?? !isVirtualHostStyleUrl(url);
const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, ''));

const accessKeyId = url.username || username || '';
Expand All @@ -402,15 +402,14 @@ export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions)
};
}

export function extractS3BucketName(s3Url: string) {
const url = new URL(s3Url);
export function extractS3BucketName(opts: S3StorageOptions) {
const url = new URL(opts.url);

if (isVirtualHostStyleUrl(url)) {
return url.hostname.split('.')[0];
if (opts.forcePathStyle || !isVirtualHostStyleUrl(url)) {
return url.pathname.slice(1);
}

// path based style
return url.pathname.slice(1);
return url.hostname.split('.')[0];
}

export function isVirtualHostStyleUrl(url: URL) {
Expand Down
2 changes: 2 additions & 0 deletions controlplane/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const {
S3_REGION,
S3_ACCESS_KEY_ID,
S3_SECRET_ACCESS_KEY,
S3_FORCE_PATH_STYLE,
SMTP_ENABLED,
SMTP_HOST,
SMTP_PORT,
Expand Down Expand Up @@ -126,6 +127,7 @@ const options: BuildConfig = {
endpoint: S3_ENDPOINT,
username: S3_ACCESS_KEY_ID,
password: S3_SECRET_ACCESS_KEY,
forcePathStyle: S3_FORCE_PATH_STYLE,
},
mailer: {
smtpEnabled: SMTP_ENABLED,
Expand Down
1 change: 1 addition & 0 deletions controlplane/src/types/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -583,4 +583,5 @@ export interface S3StorageOptions {
endpoint?: string;
username?: string;
password?: string;
forcePathStyle?: boolean;
}
Loading
Loading