From ec1ac055cdd9c2e5deb264a9c691c84fd54e9625 Mon Sep 17 00:00:00 2001 From: Andreas Zeissner Date: Tue, 3 Sep 2024 11:55:20 +0200 Subject: [PATCH] refactor: move s3 config handling to shared module --- cdn-server/package.json | 1 + cdn-server/src/s3.ts | 2 +- cdn-server/src/utils.ts | 58 ------------------- controlplane/src/core/build-server.ts | 3 +- controlplane/src/core/util.ts | 46 +-------------- controlplane/src/types/index.ts | 8 --- pnpm-lock.yaml | 35 +++++------ shared/package.json | 1 + shared/src/types/index.ts | 7 +++ shared/src/utils/util.ts | 45 ++++++++++++++ .../test/utils.s3storage.test.ts | 2 +- 11 files changed, 72 insertions(+), 136 deletions(-) delete mode 100644 cdn-server/src/utils.ts create mode 100644 shared/src/types/index.ts rename {controlplane => shared}/test/utils.s3storage.test.ts (99%) diff --git a/cdn-server/package.json b/cdn-server/package.json index 5dc762812c..3220708906 100644 --- a/cdn-server/package.json +++ b/cdn-server/package.json @@ -25,6 +25,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.529.1", "@hono/node-server": "1.11.0", + "@wundergraph/cosmo-shared": "workspace:*", "@wundergraph/cosmo-cdn": "workspace:*", "dotenv": "^16.4.5", "hono": "4.2.7" diff --git a/cdn-server/src/s3.ts b/cdn-server/src/s3.ts index eff41947a9..ba4ae54e97 100644 --- a/cdn-server/src/s3.ts +++ b/cdn-server/src/s3.ts @@ -1,7 +1,7 @@ import { GetObjectCommand, HeadObjectCommand, NoSuchKey, NotFound, S3Client } from '@aws-sdk/client-s3'; +import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared'; import { BlobNotFoundError, BlobObject, BlobStorage } from '@wundergraph/cosmo-cdn'; import { Context } from 'hono'; -import { createS3ClientConfig, extractS3BucketName } from './utils'; /** * Retrieves objects from S3 given an S3Client and a bucket name diff --git a/cdn-server/src/utils.ts b/cdn-server/src/utils.ts deleted file mode 100644 index e0ea06f2a5..0000000000 --- a/cdn-server/src/utils.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { S3ClientConfig } from '@aws-sdk/client-s3'; - -/** - * controlplane and cdn are using the same code for handling the s3 storage. - * - * see: controlplane/test/utils.s3storage.test.ts for further details - */ - -interface S3StorageOptions { - url: string; - region?: string; - endpoint?: string; - username?: string; - password?: string; -} - -export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { - const url = new URL(opts.url); - const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); - const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - - const accessKeyId = url.username || username || ''; - const secretAccessKey = url.password || password || ''; - - if (!accessKeyId || !secretAccessKey) { - throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); - } - - if (!region) { - throw new Error('Missing region in S3 configuration.'); - } - - return { - region, - endpoint, - credentials: { - accessKeyId, - secretAccessKey, - }, - forcePathStyle, - }; -} - -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); - - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; - } - - // path based style - return url.pathname.slice(1); -} - -export function isVirtualHostStyleUrl(url: URL) { - return url.hostname.split('.').length > 2; -} diff --git a/controlplane/src/core/build-server.ts b/controlplane/src/core/build-server.ts index ce2df020d6..c932d0cc21 100644 --- a/controlplane/src/core/build-server.ts +++ b/controlplane/src/core/build-server.ts @@ -2,6 +2,7 @@ import Fastify, { FastifyBaseLogger } from 'fastify'; import { S3Client } from '@aws-sdk/client-s3'; import { fastifyConnectPlugin } from '@connectrpc/connect-fastify'; import { cors, createContextValues } from '@connectrpc/connect'; +import { extractS3BucketName, createS3ClientConfig } from '@wundergraph/cosmo-shared'; import fastifyCors from '@fastify/cors'; import { pino, stdTimeFunctions, LoggerOptions } from 'pino'; import { compressionBrotli, compressionGzip } from '@connectrpc/connect-node'; @@ -37,7 +38,7 @@ import { BillingRepository } from './repositories/BillingRepository.js'; import { BillingService } from './services/BillingService.js'; import { UserRepository } from './repositories/UserRepository.js'; import { AIGraphReadmeQueue, createAIGraphReadmeWorker } from './workers/AIGraphReadmeWorker.js'; -import { fastifyLoggerId, createS3ClientConfig, extractS3BucketName } from './util.js'; +import { fastifyLoggerId } from './util.js'; import { ApiKeyRepository } from './repositories/ApiKeyRepository.js'; import { createDeleteOrganizationWorker, DeleteOrganizationQueue } from './workers/DeleteOrganizationWorker.js'; diff --git a/controlplane/src/core/util.ts b/controlplane/src/core/util.ts index 7256f5136a..dcfe82b47e 100644 --- a/controlplane/src/core/util.ts +++ b/controlplane/src/core/util.ts @@ -1,5 +1,4 @@ import { randomFill } from 'node:crypto'; -import { S3ClientConfig } from '@aws-sdk/client-s3'; import { HandlerContext } from '@connectrpc/connect'; import { EnumStatusCode, @@ -14,7 +13,7 @@ import { uid } from 'uid/secure'; import { AxiosError } from 'axios'; import { isNetworkError, isRetryableError } from 'axios-retry'; import { MemberRole, WebsocketSubprotocol } from '../db/models.js'; -import { AuthContext, DateRange, Label, ResponseMessage, S3StorageOptions } from '../types/index.js'; +import { AuthContext, DateRange, Label, ResponseMessage } from '../types/index.js'; import { isAuthenticationError, isAuthorizationError, isPublicError } from './errors/errors.js'; import { GraphKeyAuthContext } from './services/GraphApiTokenAuthenticator.js'; @@ -373,46 +372,3 @@ export function getValueOrDefault(map: Map, key: K, constructor: () export function webhookAxiosRetryCond(err: AxiosError) { return isNetworkError(err) || isRetryableError(err); } - -export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3ClientConfig { - const url = new URL(opts.url); - const { region, username, password } = opts; - const forcePathStyle = !isVirtualHostStyleUrl(url); - const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); - - const accessKeyId = url.username || username || ''; - const secretAccessKey = url.password || password || ''; - - if (!accessKeyId || !secretAccessKey) { - throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); - } - - if (!region) { - throw new Error('Missing region in S3 configuration.'); - } - - return { - region, - endpoint, - credentials: { - accessKeyId, - secretAccessKey, - }, - forcePathStyle, - }; -} - -export function extractS3BucketName(s3Url: string) { - const url = new URL(s3Url); - - if (isVirtualHostStyleUrl(url)) { - return url.hostname.split('.')[0]; - } - - // path based style - return url.pathname.slice(1); -} - -export function isVirtualHostStyleUrl(url: URL) { - return url.hostname.split('.').length > 2; -} diff --git a/controlplane/src/types/index.ts b/controlplane/src/types/index.ts index 08aeacfc58..946e49e49a 100644 --- a/controlplane/src/types/index.ts +++ b/controlplane/src/types/index.ts @@ -576,11 +576,3 @@ export interface SchemaLintIssues { warnings: LintIssueResult[]; errors: LintIssueResult[]; } - -export interface S3StorageOptions { - url: string; - region?: string; - endpoint?: string; - username?: string; - password?: string; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad0f0a8e71..7d3beaf1f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,9 @@ importers: '@wundergraph/cosmo-cdn': specifier: workspace:* version: link:cdn + '@wundergraph/cosmo-shared': + specifier: workspace:* + version: link:../shared dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -703,6 +706,9 @@ importers: shared: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.529.1 + version: 3.529.1 '@bufbuild/protobuf': specifier: ^1.9.0 version: 1.9.0 @@ -1107,7 +1113,7 @@ packages: resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==} dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.433.0 + '@aws-sdk/types': 3.523.0 tslib: 1.14.1 dev: false @@ -1115,7 +1121,7 @@ packages: resolution: {integrity: sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==} dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.433.0 + '@aws-sdk/types': 3.523.0 tslib: 1.14.1 dev: false @@ -1167,7 +1173,7 @@ packages: /@aws-crypto/util@3.0.0: resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==} dependencies: - '@aws-sdk/types': 3.433.0 + '@aws-sdk/types': 3.523.0 '@aws-sdk/util-utf8-browser': 3.259.0 tslib: 1.14.1 dev: false @@ -1663,14 +1669,6 @@ packages: - aws-crt dev: false - /@aws-sdk/types@3.433.0: - resolution: {integrity: sha512-0jEE2mSrNDd8VGFjTc1otYrwYPIkzZJEIK90ZxisKvQ/EURGBhNzWn7ejWB9XCMFT6XumYLBR0V9qq5UPisWtA==} - engines: {node: '>=14.0.0'} - dependencies: - '@smithy/types': 2.4.0 - tslib: 2.6.2 - dev: false - /@aws-sdk/types@3.523.0: resolution: {integrity: sha512-AqGIu4u+SxPiUuNBp2acCVcq80KDUFjxe6e3cMTvKWTzCbrVk1AXv0dAaJnCmdkWIha6zJDWxpIk/aL4EGhZ9A==} engines: {node: '>=14.0.0'} @@ -11544,13 +11542,6 @@ packages: tslib: 2.6.2 dev: false - /@smithy/types@2.4.0: - resolution: {integrity: sha512-iH1Xz68FWlmBJ9vvYeHifVMWJf82ONx+OybPW8ZGf5wnEv2S0UXcU4zwlwJkRXuLKpcSLHrraHbn2ucdVXLb4g==} - engines: {node: '>=14.0.0'} - dependencies: - tslib: 2.6.2 - dev: false - /@smithy/url-parser@2.1.3: resolution: {integrity: sha512-X1NRA4WzK/ihgyzTpeGvI9Wn45y8HmqF4AZ/FazwAv8V203Ex+4lXqcYI70naX9ETqbqKVzFk88W6WJJzCggTQ==} dependencies: @@ -12692,7 +12683,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/type-utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) '@typescript-eslint/utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.52.0 graphemer: 1.4.0 ignore: 5.2.4 @@ -12793,7 +12784,7 @@ packages: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.5.2) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.52.0 typescript: 5.5.2 transitivePeerDependencies: @@ -12860,7 +12851,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.5.2) '@typescript-eslint/utils': 5.62.0(eslint@8.52.0)(typescript@5.5.2) - debug: 4.3.5 + debug: 4.3.4 eslint: 8.52.0 tsutils: 3.21.0(typescript@5.5.2) typescript: 5.5.2 @@ -16206,7 +16197,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.5 + debug: 4.3.4 enhanced-resolve: 5.15.0 eslint: 8.52.0 eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0)(eslint-import-resolver-typescript@3.5.5)(eslint@8.52.0) diff --git a/shared/package.json b/shared/package.json index bdf24028af..b78f2b4680 100644 --- a/shared/package.json +++ b/shared/package.json @@ -33,6 +33,7 @@ "license": "Apache-2.0", "dependencies": { "@bufbuild/protobuf": "^1.9.0", + "@aws-sdk/client-s3": "^3.529.1", "@graphql-tools/schema": "^8.5.1", "@graphql-tools/utils": "^9.2.1", "@wundergraph/composition": "workspace:*", diff --git a/shared/src/types/index.ts b/shared/src/types/index.ts new file mode 100644 index 0000000000..bce07f22fe --- /dev/null +++ b/shared/src/types/index.ts @@ -0,0 +1,7 @@ +export interface S3StorageOptions { + url: string; + region?: string; + endpoint?: string; + username?: string; + password?: string; +} diff --git a/shared/src/utils/util.ts b/shared/src/utils/util.ts index 9acb0f9e3e..f22a701141 100644 --- a/shared/src/utils/util.ts +++ b/shared/src/utils/util.ts @@ -1,4 +1,6 @@ +import * as S3 from '@aws-sdk/client-s3'; import { SubscriptionProtocol, WebsocketSubprotocol } from '../router-config/builder.js'; +import { S3StorageOptions } from '../types/index.js'; export function delay(t: number) { return new Promise((resolve) => setTimeout(resolve, t)); @@ -76,3 +78,46 @@ export function isValidWebsocketSubprotocol(protocol: WebsocketSubprotocol) { } } } + +export function createS3ClientConfig(bucketName: string, opts: S3StorageOptions): S3.S3ClientConfig { + const url = new URL(opts.url); + const { region, username, password } = opts; + const forcePathStyle = !isVirtualHostStyleUrl(url); + const endpoint = opts.endpoint || (forcePathStyle ? url.origin : url.origin.replace(`${bucketName}.`, '')); + + const accessKeyId = url.username || username || ''; + const secretAccessKey = url.password || password || ''; + + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing S3 credentials. Please provide access key ID and secret access key.'); + } + + if (!region) { + throw new Error('Missing region in S3 configuration.'); + } + + return { + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle, + }; +} + +export function extractS3BucketName(s3Url: string) { + const url = new URL(s3Url); + + if (isVirtualHostStyleUrl(url)) { + return url.hostname.split('.')[0]; + } + + // path based style + return url.pathname.slice(1); +} + +export function isVirtualHostStyleUrl(url: URL) { + return url.hostname.split('.').length > 2; +} diff --git a/controlplane/test/utils.s3storage.test.ts b/shared/test/utils.s3storage.test.ts similarity index 99% rename from controlplane/test/utils.s3storage.test.ts rename to shared/test/utils.s3storage.test.ts index 7919e2a3bd..d197619297 100644 --- a/controlplane/test/utils.s3storage.test.ts +++ b/shared/test/utils.s3storage.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src/core/util.js'; +import { createS3ClientConfig, extractS3BucketName, isVirtualHostStyleUrl } from '../src'; describe('S3 Utils', () => { describe('createS3ClientConfig', () => {