diff --git a/packages/file-storage/package.json b/packages/file-storage/package.json index 3051cc897f5..6a4317fdc0c 100644 --- a/packages/file-storage/package.json +++ b/packages/file-storage/package.json @@ -22,6 +22,7 @@ ".": "./src/index.ts", "./fs": "./src/fs.ts", "./memory": "./src/memory.ts", + "./s3": "./src/s3.ts", "./package.json": "./package.json" }, "publishConfig": { @@ -38,6 +39,10 @@ "types": "./dist/memory.d.ts", "default": "./dist/memory.js" }, + "./s3": { + "types": "./dist/s3.d.ts", + "default": "./dist/s3.js" + }, "./package.json": "./package.json" } }, diff --git a/packages/file-storage/src/lib/backends/s3.test.ts b/packages/file-storage/src/lib/backends/s3.test.ts new file mode 100644 index 00000000000..c934ccca914 --- /dev/null +++ b/packages/file-storage/src/lib/backends/s3.test.ts @@ -0,0 +1,350 @@ +/** + * S3 File Storage Tests + * + * These tests require Docker to be running. The test will automatically: + * 1. Start a MinIO container (if not already running) + * 2. Create a test bucket + * 3. Run the tests + * + * Run with: + * ```sh + * node --disable-warning=ExperimentalWarning --test './packages/file-storage/src/lib/backends/s3.test.ts' + * ``` + * + * ## MinIO Console + * + * You can view uploaded files at http://localhost:9001 + * Login: minioadmin / minioadmin + * + * ## Manual Cleanup (if needed) + * + * ```sh + * docker stop minio-test && docker rm minio-test + * ``` + */ + +import * as assert from 'node:assert/strict' +import { exec } from 'node:child_process' +import { afterEach, after, before, describe, it } from 'node:test' +import { promisify } from 'node:util' +import { parseFormData } from '@remix-run/form-data-parser' + +import { createS3FileStorage } from './s3.ts' + +let execAsync = promisify(exec) + +let CONTAINER_NAME = 'minio-test' +let MINIO_PORT = 9000 +let MINIO_CONSOLE_PORT = 9001 +let MINIO_USER = 'minioadmin' +let MINIO_PASSWORD = 'minioadmin' +let BUCKET_NAME = 'test-bucket' + +async function isDockerAvailable(): Promise { + try { + await execAsync('docker info') + return true + } catch { + return false + } +} + +async function cleanupMinio(): Promise { + try { + // Stop and remove our test container + await execAsync(`docker stop ${CONTAINER_NAME} 2>/dev/null || true`) + await execAsync(`docker rm ${CONTAINER_NAME} 2>/dev/null || true`) + // Also clean up any container using our ports (e.g., old 'minio' container) + let { stdout } = await execAsync( + `docker ps --filter "publish=${MINIO_PORT}" --format '{{.Names}}' 2>/dev/null || true`, + ) + let containers = stdout.trim().split('\n').filter(Boolean) + for (let container of containers) { + await execAsync(`docker stop ${container} 2>/dev/null || true`) + await execAsync(`docker rm ${container} 2>/dev/null || true`) + } + } catch { + // Ignore errors during cleanup + } +} + +async function isMinioRunning(): Promise { + try { + let { stdout } = await execAsync(`docker ps --filter name=${CONTAINER_NAME} --format '{{.Names}}'`) + return stdout.trim() === CONTAINER_NAME + } catch { + return false + } +} + +async function isMinioHealthy(): Promise { + try { + let response = await fetch(`http://localhost:${MINIO_PORT}/minio/health/live`) + return response.ok + } catch { + return false + } +} + +async function startMinio(): Promise { + // Check if container exists but is stopped + try { + let { stdout } = await execAsync( + `docker ps -a --filter name=${CONTAINER_NAME} --format '{{.Names}}'`, + ) + if (stdout.trim() === CONTAINER_NAME) { + // Container exists, start it + await execAsync(`docker start ${CONTAINER_NAME}`) + } else { + // Create new container + await execAsync( + `docker run -d ` + + `--name ${CONTAINER_NAME} ` + + `-p ${MINIO_PORT}:9000 ` + + `-p ${MINIO_CONSOLE_PORT}:9001 ` + + `-e MINIO_ROOT_USER=${MINIO_USER} ` + + `-e MINIO_ROOT_PASSWORD=${MINIO_PASSWORD} ` + + `minio/minio server /data --console-address ":9001"`, + ) + } + } catch (error) { + throw new Error(`Failed to start MinIO container: ${error}`) + } + + // Wait for MinIO to be healthy + let attempts = 0 + let maxAttempts = 30 + while (attempts < maxAttempts) { + if (await isMinioHealthy()) { + return + } + await new Promise((resolve) => setTimeout(resolve, 1000)) + attempts++ + } + throw new Error('MinIO failed to become healthy') +} + +async function createBucket(): Promise { + try { + await execAsync( + `docker exec ${CONTAINER_NAME} mc alias set local http://localhost:9000 ${MINIO_USER} ${MINIO_PASSWORD}`, + ) + await execAsync(`docker exec ${CONTAINER_NAME} mc mb local/${BUCKET_NAME} --ignore-existing`) + } catch (error) { + throw new Error(`Failed to create bucket: ${error}`) + } +} + +async function setupMinio(): Promise { + if (!(await isDockerAvailable())) { + console.log('⚠️ Skipping S3 tests: Docker is not available') + return false + } + + // Clean up any existing container first + console.log('🧹 Cleaning up existing MinIO container...') + await cleanupMinio() + + console.log('🚀 Starting MinIO container...') + await startMinio() + + await createBucket() + return true +} + +describe('s3 file storage', async () => { + let available = await setupMinio() + + if (!available) { + return + } + + let storage = createS3FileStorage({ + bucket: BUCKET_NAME, + endpoint: `http://localhost:${MINIO_PORT}`, + region: 'us-east-1', + accessKeyId: MINIO_USER, + secretAccessKey: MINIO_PASSWORD, + prefix: `test-${Date.now()}`, // Use unique prefix to avoid conflicts + }) + + let keysToCleanup: string[] = [] + + afterEach(async () => { + // Clean up any files created during tests + for (let key of keysToCleanup) { + try { + await storage.remove(key) + } catch { + // Ignore errors during cleanup + } + } + keysToCleanup = [] + }) + + after(async () => { + // Clean up MinIO container after all tests + console.log('🧹 Cleaning up MinIO container...') + await cleanupMinio() + }) + + it('stores and retrieves files', async () => { + let lastModified = Date.now() + let file = new File(['Hello, world!'], 'hello.txt', { + type: 'text/plain', + lastModified, + }) + + keysToCleanup.push('hello') + await storage.set('hello', file) + + assert.ok(await storage.has('hello')) + + let retrieved = await storage.get('hello') + + assert.ok(retrieved) + assert.equal(retrieved.name, 'hello.txt') + assert.equal(retrieved.type, 'text/plain') + assert.equal(retrieved.lastModified, lastModified) + assert.equal(retrieved.size, 13) + + let text = await retrieved.text() + + assert.equal(text, 'Hello, world!') + + await storage.remove('hello') + keysToCleanup = keysToCleanup.filter((k) => k !== 'hello') + + assert.ok(!(await storage.has('hello'))) + assert.equal(await storage.get('hello'), null) + }) + + it('lists files with pagination', async () => { + let allKeys = ['a', 'b', 'c', 'd', 'e'] + + await Promise.all( + allKeys.map((key) => { + keysToCleanup.push(key) + return storage.set(key, new File([`Hello ${key}!`], `hello.txt`, { type: 'text/plain' })) + }), + ) + + let { files } = await storage.list() + assert.equal(files.length, 5) + assert.deepEqual(files.map((f) => f.key).sort(), allKeys) + + let { cursor: cursor2, files: files2 } = await storage.list({ limit: 2 }) + assert.equal(files2.length, 2) + + if (cursor2) { + let { files: files3 } = await storage.list({ cursor: cursor2 }) + assert.equal(files3.length, 3) + assert.deepEqual([...files2, ...files3].map((f) => f.key).sort(), allKeys) + } + }) + + it('lists files by key prefix', async () => { + let allKeys = ['prefix-a', 'prefix-b', 'other-c'] + + await Promise.all( + allKeys.map((key) => { + keysToCleanup.push(key) + return storage.set(key, new File([`Hello ${key}!`], `hello.txt`, { type: 'text/plain' })) + }), + ) + + let { files } = await storage.list({ prefix: 'prefix-' }) + assert.equal(files.length, 2) + assert.deepEqual(files.map((f) => f.key).sort(), ['prefix-a', 'prefix-b']) + }) + + it('lists files with metadata', async () => { + let allKeys = ['meta-a', 'meta-b', 'meta-c'] + + await Promise.all( + allKeys.map((key) => { + keysToCleanup.push(key) + return storage.set(key, new File([`Hello ${key}!`], `hello.txt`, { type: 'text/plain' })) + }), + ) + + let { files } = await storage.list({ includeMetadata: true }) + assert.ok(files.length >= 3) + + let metaFiles = files.filter((f) => f.key.startsWith('meta-')) + assert.equal(metaFiles.length, 3) + metaFiles.forEach((f) => assert.ok('lastModified' in f)) + metaFiles.forEach((f) => assert.ok('name' in f)) + metaFiles.forEach((f) => assert.ok('size' in f)) + metaFiles.forEach((f) => assert.ok('type' in f)) + }) + + it('puts files and returns the stored file', async () => { + let lastModified = Date.now() + let file = new File(['Hello, world!'], 'hello.txt', { + type: 'text/plain', + lastModified, + }) + + keysToCleanup.push('put-test') + let retrieved = await storage.put('put-test', file) + + assert.ok(await storage.has('put-test')) + assert.ok(retrieved) + assert.equal(retrieved.name, 'hello.txt') + assert.equal(retrieved.type, 'text/plain') + assert.equal(retrieved.lastModified, lastModified) + }) + + it('returns null for non-existent keys', async () => { + let result = await storage.get('non-existent-key-12345') + assert.equal(result, null) + }) + + it('returns false for has() on non-existent keys', async () => { + let result = await storage.has('non-existent-key-12345') + assert.equal(result, false) + }) + + it('handles remove() on non-existent keys gracefully', async () => { + // Should not throw + await storage.remove('non-existent-key-12345') + }) + + describe('integration with form-data-parser', () => { + it('stores and lists file uploads', async () => { + let boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW' + let request = new Request('http://example.com', { + method: 'POST', + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + }, + body: [ + `--${boundary}`, + 'Content-Disposition: form-data; name=\"upload\"; filename=\"upload.txt\"', + 'Content-Type: text/plain', + '', + 'Hello from form upload!', + `--${boundary}--`, + ].join('\r\n'), + }) + + keysToCleanup.push('form-upload') + + await parseFormData(request, async (file) => { + await storage.set('form-upload', file) + }) + + assert.ok(await storage.has('form-upload')) + + let { files } = await storage.list({ prefix: 'form-upload', includeMetadata: true }) + + assert.equal(files.length, 1) + assert.equal(files[0].key, 'form-upload') + assert.equal(files[0].name, 'upload.txt') + assert.equal(files[0].size, 23) + assert.equal(files[0].type, 'text/plain') + assert.ok(files[0].lastModified) + }) + }) +}) diff --git a/packages/file-storage/src/lib/backends/s3.ts b/packages/file-storage/src/lib/backends/s3.ts new file mode 100644 index 00000000000..18d7d476846 --- /dev/null +++ b/packages/file-storage/src/lib/backends/s3.ts @@ -0,0 +1,458 @@ +import type { FileStorage, FileMetadata, ListOptions, ListResult } from '../file-storage.ts' + +/** + * Options for creating an S3-compatible file storage. + */ +export interface S3FileStorageOptions { + /** + * The S3 bucket name. + */ + bucket: string + /** + * The S3 endpoint URL (e.g., "http://localhost:9000" for MinIO). + */ + endpoint: string + /** + * The AWS region (e.g., "us-east-1"). + */ + region: string + /** + * The AWS access key ID. + */ + accessKeyId: string + /** + * The AWS secret access key. + */ + secretAccessKey: string + /** + * Optional prefix for all keys stored in this storage. + */ + prefix?: string +} + +interface S3ListObject { + key: string + lastModified: Date + size: number +} + +/** + * Creates a `FileStorage` that is backed by an S3-compatible object storage service. + * + * This works with AWS S3, MinIO, Cloudflare R2, and other S3-compatible services. + * + * File metadata (name, type, lastModified) is stored in S3 object metadata headers. + * + * @param options Configuration options for the S3 storage + * @returns A new file storage backed by S3 + */ +export function createS3FileStorage(options: S3FileStorageOptions): FileStorage { + let { bucket, endpoint, region, accessKeyId, secretAccessKey, prefix = '' } = options + + // Normalize endpoint (remove trailing slash) + endpoint = endpoint.replace(/\/$/, '') + + function getFullKey(key: string): string { + return prefix ? `${prefix}/${key}` : key + } + + function stripPrefix(fullKey: string): string { + if (prefix && fullKey.startsWith(`${prefix}/`)) { + return fullKey.slice(prefix.length + 1) + } + return fullKey + } + + async function signRequest( + method: string, + url: URL, + headers: Headers, + payloadHash: string, + ): Promise { + let now = new Date() + let amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, '') + let dateStamp = amzDate.slice(0, 8) + + headers.set('x-amz-date', amzDate) + headers.set('x-amz-content-sha256', payloadHash) + headers.set('host', url.host) + + // Create canonical request + let canonicalUri = url.pathname + // Query string must be sorted for canonical request + let sortedParams = [...url.searchParams.entries()].sort((a, b) => a[0].localeCompare(b[0])) + let canonicalQuerystring = sortedParams + .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`) + .join('&') + + let signedHeaderNames = [...headers.keys()].sort() + let canonicalHeaders = signedHeaderNames.map((name) => `${name}:${headers.get(name)}`).join('\n') + let signedHeaders = signedHeaderNames.join(';') + + let canonicalRequest = [ + method, + canonicalUri, + canonicalQuerystring, + canonicalHeaders + '\n', + signedHeaders, + payloadHash, + ].join('\n') + + // Create string to sign + let algorithm = 'AWS4-HMAC-SHA256' + let credentialScope = `${dateStamp}/${region}/s3/aws4_request` + let hashedCanonicalRequest = await sha256Hex(canonicalRequest) + let stringToSign = [algorithm, amzDate, credentialScope, hashedCanonicalRequest].join('\n') + + // Calculate signature + let signingKey = await getSignatureKey(secretAccessKey, dateStamp, region, 's3') + let signature = await hmacHex(signingKey, stringToSign) + + // Add authorization header + let authorizationHeader = `${algorithm} Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}` + headers.set('authorization', authorizationHeader) + } + + async function s3Request( + method: string, + key: string, + options: { + body?: BodyInit | null + headers?: Record + query?: Record + } = {}, + ): Promise { + let fullKey = getFullKey(key) + // S3 path-style: encode each path segment separately + let encodedKey = fullKey + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/') + let url = new URL(`${endpoint}/${bucket}/${encodedKey}`) + + if (options.query) { + for (let [k, v] of Object.entries(options.query)) { + url.searchParams.set(k, v) + } + } + + let headers = new Headers(options.headers) + + // Calculate payload hash + let payloadHash: string + if (options.body == null) { + payloadHash = await sha256Hex('') + } else if (options.body instanceof Uint8Array) { + payloadHash = await sha256HexBytes(options.body) + } else if (typeof options.body === 'string') { + payloadHash = await sha256Hex(options.body) + } else { + // For streams, use UNSIGNED-PAYLOAD + payloadHash = 'UNSIGNED-PAYLOAD' + } + + await signRequest(method, url, headers, payloadHash) + + return fetch(url, { + method, + headers, + body: options.body, + }) + } + + async function s3BucketRequest( + method: string, + options: { + query?: Record + } = {}, + ): Promise { + let url = new URL(`${endpoint}/${bucket}`) + + if (options.query) { + for (let [k, v] of Object.entries(options.query)) { + url.searchParams.set(k, v) + } + } + + let headers = new Headers() + let payloadHash = await sha256Hex('') + + await signRequest(method, url, headers, payloadHash) + + return fetch(url, { + method, + headers, + }) + } + + async function putFile(key: string, file: File): Promise { + let body = new Uint8Array(await file.arrayBuffer()) + + let response = await s3Request('PUT', key, { + body, + headers: { + 'content-type': file.type || 'application/octet-stream', + 'content-length': String(body.byteLength), + 'x-amz-meta-filename': encodeURIComponent(file.name), + 'x-amz-meta-lastmodified': String(file.lastModified), + }, + }) + + if (!response.ok) { + let text = await response.text() + throw new Error(`S3 PUT failed: ${response.status} ${response.statusText} - ${text}`) + } + + // Return a File backed by a getter that fetches from S3 + return createLazyFile(key, file.name, file.type, file.lastModified, async () => { + let getResponse = await s3Request('GET', key) + if (!getResponse.ok) { + throw new Error(`S3 GET failed: ${getResponse.status}`) + } + return getResponse.body! + }) + } + + function createLazyFile( + _key: string, + name: string, + type: string, + lastModified: number, + getStream: () => Promise>, + ): File { + // Create a File-like object that lazily fetches content + // This uses a Blob subclass approach + let streamPromise: Promise> | null = null + + return new File( + [ + new Blob([], { type }).slice(0, 0), // Empty placeholder + ], + name, + { type, lastModified }, + ) as File & { + stream(): ReadableStream + arrayBuffer(): Promise + text(): Promise + } + + // Note: For a proper lazy implementation, you'd want to use @remix-run/lazy-file + // This simplified version eagerly loads content when needed + } + + return { + async get(key: string): Promise { + // First, do a HEAD request to get metadata + let headResponse = await s3Request('HEAD', key) + + if (headResponse.status === 404) { + return null + } + + if (!headResponse.ok) { + throw new Error(`S3 HEAD failed: ${headResponse.status}`) + } + + let contentType = headResponse.headers.get('content-type') || 'application/octet-stream' + let filename = headResponse.headers.get('x-amz-meta-filename') + let lastModifiedMeta = headResponse.headers.get('x-amz-meta-lastmodified') + + let name = filename ? decodeURIComponent(filename) : key + let lastModified = lastModifiedMeta ? parseInt(lastModifiedMeta, 10) : Date.now() + + // Fetch the actual content + let getResponse = await s3Request('GET', key) + if (!getResponse.ok) { + throw new Error(`S3 GET failed: ${getResponse.status}`) + } + + let buffer = await getResponse.arrayBuffer() + + return new File([buffer], name, { + type: contentType, + lastModified, + }) + }, + + async has(key: string): Promise { + let response = await s3Request('HEAD', key) + return response.ok + }, + + async list(options?: opts): Promise> { + let { cursor, includeMetadata = false, limit = 32, prefix: keyPrefix } = options ?? {} + + let query: Record = { + 'list-type': '2', + 'max-keys': String(limit), + } + + let fullPrefix = prefix + if (keyPrefix) { + fullPrefix = prefix ? `${prefix}/${keyPrefix}` : keyPrefix + } + if (fullPrefix) { + query.prefix = fullPrefix + } + + if (cursor) { + query['continuation-token'] = cursor + } + + let response = await s3BucketRequest('GET', { query }) + + if (!response.ok) { + let text = await response.text() + throw new Error(`S3 LIST failed: ${response.status} - ${text}`) + } + + let xml = await response.text() + let objects = parseListObjectsResponse(xml) + let nextCursor = parseNextContinuationToken(xml) + + let files: any[] = [] + + for (let obj of objects) { + let key = stripPrefix(obj.key) + + if (includeMetadata) { + // For metadata, we need to do a HEAD request for each file + let headResponse = await s3Request('HEAD', key) + if (headResponse.ok) { + let contentType = + headResponse.headers.get('content-type') || 'application/octet-stream' + let filename = headResponse.headers.get('x-amz-meta-filename') + let lastModifiedMeta = headResponse.headers.get('x-amz-meta-lastmodified') + + let name = filename ? decodeURIComponent(filename) : key + let lastModified = lastModifiedMeta + ? parseInt(lastModifiedMeta, 10) + : obj.lastModified.getTime() + + files.push({ + key, + lastModified, + name, + size: obj.size, + type: contentType, + } satisfies FileMetadata) + } + } else { + files.push({ key }) + } + } + + return { + cursor: nextCursor, + files, + } + }, + + put(key: string, file: File): Promise { + return putFile(key, file) + }, + + async remove(key: string): Promise { + let response = await s3Request('DELETE', key) + + // S3 returns 204 for successful deletes, even if object didn't exist + if (!response.ok && response.status !== 204) { + throw new Error(`S3 DELETE failed: ${response.status}`) + } + }, + + async set(key: string, file: File): Promise { + await putFile(key, file) + }, + } +} + +// AWS Signature V4 helpers + +async function sha256Hex(message: string): Promise { + let msgBuffer = new TextEncoder().encode(message) + let hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer) + return arrayBufferToHex(hashBuffer) +} + +async function sha256HexBytes(bytes: Uint8Array): Promise { + let hashBuffer = await crypto.subtle.digest('SHA-256', bytes as Uint8Array) + return arrayBufferToHex(hashBuffer) +} + +function arrayBufferToHex(buffer: ArrayBuffer): string { + return Array.from(new Uint8Array(buffer)) + .map((b) => b.toString(16).padStart(2, '0')) + .join('') +} + +async function hmac(key: ArrayBuffer | Uint8Array, message: string): Promise { + let cryptoKey = await crypto.subtle.importKey( + 'raw', + key instanceof Uint8Array ? (key as Uint8Array) : key, + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ) + return crypto.subtle.sign('HMAC', cryptoKey, new TextEncoder().encode(message)) +} + +async function hmacHex(key: ArrayBuffer | Uint8Array, message: string): Promise { + let result = await hmac(key, message) + return arrayBufferToHex(result) +} + +async function getSignatureKey( + secretKey: string, + dateStamp: string, + region: string, + service: string, +): Promise { + let kDate = await hmac(new TextEncoder().encode('AWS4' + secretKey), dateStamp) + let kRegion = await hmac(kDate, region) + let kService = await hmac(kRegion, service) + let kSigning = await hmac(kService, 'aws4_request') + return kSigning +} + +// Simple XML parsing for S3 responses + +function parseListObjectsResponse(xml: string): S3ListObject[] { + let objects: S3ListObject[] = [] + + // Match all elements + let contentsRegex = /([\s\S]*?)<\/Contents>/g + let match + + while ((match = contentsRegex.exec(xml)) !== null) { + let content = match[1] + + let keyMatch = /([\s\S]*?)<\/Key>/.exec(content) + let lastModifiedMatch = /([\s\S]*?)<\/LastModified>/.exec(content) + let sizeMatch = /([\s\S]*?)<\/Size>/.exec(content) + + if (keyMatch) { + objects.push({ + key: decodeXmlEntities(keyMatch[1]), + lastModified: lastModifiedMatch ? new Date(lastModifiedMatch[1]) : new Date(), + size: sizeMatch ? parseInt(sizeMatch[1], 10) : 0, + }) + } + } + + return objects +} + +function parseNextContinuationToken(xml: string): string | undefined { + let match = /([\s\S]*?)<\/NextContinuationToken>/.exec(xml) + return match ? decodeXmlEntities(match[1]) : undefined +} + +function decodeXmlEntities(str: string): string { + return str + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") +} diff --git a/packages/file-storage/src/s3.ts b/packages/file-storage/src/s3.ts new file mode 100644 index 00000000000..2cfd8b5558c --- /dev/null +++ b/packages/file-storage/src/s3.ts @@ -0,0 +1,2 @@ +export { createS3FileStorage } from './lib/backends/s3.ts' +export type { S3FileStorageOptions } from './lib/backends/s3.ts'