From 5990033aa48a553f567edcb23babbc1f3a47ff77 Mon Sep 17 00:00:00 2001 From: "Marc J. Schmidt" Date: Tue, 10 Oct 2023 17:02:35 +0200 Subject: [PATCH] feature(storage): add publicUrl support --- packages/storage-aws-s3/README.md | 15 +--- packages/storage-aws-s3/src/s3-adapter.ts | 78 +++++++++++++++---- packages/storage-aws-s3/tests/storage.spec.ts | 14 +++- packages/storage-ftp/src/ftp-adapter.ts | 6 +- packages/storage/src/local-adapter.ts | 15 ++-- packages/storage/src/memory-adapter.ts | 12 ++- packages/storage/src/storage.ts | 56 +++++++++++-- packages/storage/tests/local.spec.ts | 9 +-- packages/storage/tests/storage.spec.ts | 56 +++++++++---- 9 files changed, 194 insertions(+), 67 deletions(-) diff --git a/packages/storage-aws-s3/README.md b/packages/storage-aws-s3/README.md index dbd412bc1..47d510fd8 100644 --- a/packages/storage-aws-s3/README.md +++ b/packages/storage-aws-s3/README.md @@ -1,14 +1,5 @@ -# skeleton package +# Deepkit Storage AWS S3 -This package can be copied when a new package should be created. +This package provides Deepkit Storage integration for AWS S3. -### Steps after copying: - -- Adjust "name", "description", and "private" in `package.json`. -- Adjust README.md -- Put this package into root `/package.json` "jest.references". -- Put this package into root `/tsconfig.json` "references". -- Put this package into root `/tsconfig.esm.json` "references". -- Add dependencies to `package.json` and run `node sync-tsconfig-deps.js` to adjust tsconfig automatically. -- Add to .github/workflows/main.yml tsc build step if necessary. -- Add to typedoc build in deepkit-website if necessary. +If you create a S3 bucket and want (public) visibility support, make sure ACLs is enabled and "Block public access" is disabled. diff --git a/packages/storage-aws-s3/src/s3-adapter.ts b/packages/storage-aws-s3/src/s3-adapter.ts index 9c9e90c91..d85209244 100644 --- a/packages/storage-aws-s3/src/s3-adapter.ts +++ b/packages/storage-aws-s3/src/s3-adapter.ts @@ -2,12 +2,13 @@ import { FileType, FileVisibility, pathDirectory, Reporter, StorageAdapter, Stor import { CopyObjectCommand, DeleteObjectsCommand, + GetObjectAclCommand, GetObjectCommand, HeadObjectCommand, ListObjectsCommand, - ListObjectsV2Command, + PutObjectAclCommand, PutObjectCommand, - S3Client + S3Client, } from '@aws-sdk/client-s3'; import { normalizePath } from 'typedoc'; @@ -24,7 +25,7 @@ export interface StorageAwsS3Options { * * Default: true */ - directorySupport?: boolean; + directoryEmulation?: boolean; } export class StorageAwsS3Adapter implements StorageAdapter { @@ -45,8 +46,8 @@ export class StorageAwsS3Adapter implements StorageAdapter { return true; } - protected isDirectorySupport(): boolean { - return this.options.directorySupport !== false; + supportsDirectory() { + return this.options.directoryEmulation === true; } protected getRemotePath(path: string) { @@ -69,17 +70,33 @@ export class StorageAwsS3Adapter implements StorageAdapter { return path.slice(base.length); } - async url(path: string): Promise { - return `s3://${this.options.bucket}/${this.getRemotePath(path)}`; + async publicUrl(path: string): Promise { + return `https://${this.options.bucket}.s3.${this.options.region}.amazonaws.com/${this.getRemotePath(path)}`; } - async makeDirectory(path: string): Promise { + protected visibilityToAcl(visibility: FileVisibility): string { + if (visibility === 'public') return 'public-read'; + return 'private'; + } + + protected aclToVisibility(grants: any[]): FileVisibility { + for (const grant of grants) { + if (grant.Permission === 'READ' && grant.Grantee.URI === 'http://acs.amazonaws.com/groups/global/AllUsers') { + return 'public'; + } + } + + return 'private'; + } + + async makeDirectory(path: string, visibility: FileVisibility): Promise { if (path === '') return; const command = new PutObjectCommand({ Bucket: this.options.bucket, Key: this.getRemoteDirectory(path), - ContentLength: 0 + ContentLength: 0, + ACL: this.visibilityToAcl(visibility), }); try { @@ -101,8 +118,7 @@ export class StorageAwsS3Adapter implements StorageAdapter { const files: StorageFile[] = []; const remotePath = this.getRemoteDirectory(path); - //only v2 includes directories - const command = new ListObjectsV2Command({ + const command = new ListObjectsCommand({ Bucket: this.options.bucket, Prefix: remotePath, Delimiter: recursive ? undefined : '/', @@ -144,8 +160,8 @@ export class StorageAwsS3Adapter implements StorageAdapter { const file = await this.get(source); if (file && file.isFile()) { - if (this.isDirectorySupport()) { - await this.makeDirectory(pathDirectory(destination)); + if (this.supportsDirectory()) { + await this.makeDirectory(pathDirectory(destination), file.visibility); } const command = new CopyObjectCommand({ @@ -252,6 +268,7 @@ export class StorageAwsS3Adapter implements StorageAdapter { const response = await this.client.send(command); file.size = response.ContentLength || 0; file.lastModified = response.LastModified; + file.visibility = await this.getVisibility(path); } catch (error: any) { return undefined; } @@ -283,8 +300,8 @@ export class StorageAwsS3Adapter implements StorageAdapter { } async write(path: string, contents: Uint8Array, visibility: FileVisibility, reporter: Reporter): Promise { - if (this.isDirectorySupport()) { - await this.makeDirectory(pathDirectory(path)); + if (this.supportsDirectory()) { + await this.makeDirectory(pathDirectory(path), visibility); } const remotePath = this.getRemotePath(path); @@ -292,6 +309,7 @@ export class StorageAwsS3Adapter implements StorageAdapter { Bucket: this.options.bucket, Key: remotePath, Body: contents, + ACL: this.visibilityToAcl(visibility), }); try { @@ -300,4 +318,34 @@ export class StorageAwsS3Adapter implements StorageAdapter { throw new StorageError(`Could not write file ${path}: ${error.message}`); } } + + async getVisibility(path: string): Promise { + const remotePath = this.getRemotePath(path); + const aclCommand = new GetObjectAclCommand({ + Bucket: this.options.bucket, + Key: remotePath, + }); + + try { + const response = await this.client.send(aclCommand); + return this.aclToVisibility(response.Grants || []); + } catch (error: any) { + throw new StorageError(`Could not get visibility for ${path}: ${error.message}`); + } + } + + async setVisibility(path: string, visibility: FileVisibility): Promise { + const remotePath = this.getRemotePath(path); + const command = new PutObjectAclCommand({ + Bucket: this.options.bucket, + Key: remotePath, + ACL: this.visibilityToAcl(visibility), + }); + + try { + await this.client.send(command); + } catch (error: any) { + throw new StorageError(`Could not set visibility for ${path}: ${error.message}`); + } + } } diff --git a/packages/storage-aws-s3/tests/storage.spec.ts b/packages/storage-aws-s3/tests/storage.spec.ts index 647d7ce4b..896f26334 100644 --- a/packages/storage-aws-s3/tests/storage.spec.ts +++ b/packages/storage-aws-s3/tests/storage.spec.ts @@ -1,8 +1,9 @@ -import { test } from '@jest/globals'; +import { expect, test } from '@jest/globals'; import './storage.spec.js'; -import { setAdapterFactory } from '@deepkit/storage/test'; +import { adapterFactory, setAdapterFactory } from '@deepkit/storage/test'; import { StorageAwsS3Adapter } from '../src/s3-adapter.js'; import { DeleteObjectsCommand, ListObjectsCommand } from '@aws-sdk/client-s3'; +import { Storage } from '@deepkit/storage'; setAdapterFactory(async () => { const folder = 'test-folder-dont-delete'; @@ -38,11 +39,20 @@ setAdapterFactory(async () => { return adapter; }); +test('s3 url', async () => { + const storage = new Storage(await adapterFactory()); + + await storage.write('test.txt', 'abc', 'public'); + const url = await storage.publicUrl('test.txt'); + expect(url).toBe('https://deepkit-storage-integration-tests.s3.eu-central-1.amazonaws.com/test-folder-dont-delete/test.txt'); +}); + // since we import .storage.spec.js, all its tests are scheduled to run // we define 'basic' here too, so we can easily run just this test. // also necessary to have at least once test in this file, so that WebStorm // detects the file as a test file. test('basic', () => undefined); +test('visibility', () => undefined); test('recursive', () => undefined); test('copy', () => undefined); test('move', () => undefined); diff --git a/packages/storage-ftp/src/ftp-adapter.ts b/packages/storage-ftp/src/ftp-adapter.ts index f3acec83c..90251c882 100644 --- a/packages/storage-ftp/src/ftp-adapter.ts +++ b/packages/storage-ftp/src/ftp-adapter.ts @@ -54,6 +54,10 @@ export class StorageFtpAdapter implements StorageAdapter { return false; } + supportsDirectory() { + return true; + } + async clearWorkingDir() { await this.ensureConnected(); await this.client.clearWorkingDir(); @@ -64,7 +68,7 @@ export class StorageFtpAdapter implements StorageAdapter { return resolveStoragePath([this.options.root, path]); } - async url(path: string): Promise { + async publicUrl(path: string): Promise { return `ftp://${this.options.host}:${this.options.port}/${this.getRemotePath(path)}`; } diff --git a/packages/storage/src/local-adapter.ts b/packages/storage/src/local-adapter.ts index 26bf39183..b96cdb18d 100644 --- a/packages/storage/src/local-adapter.ts +++ b/packages/storage/src/local-adapter.ts @@ -3,7 +3,6 @@ import type * as fs from 'fs/promises'; export interface StorageLocalAdapterOptions { root: string; - url: string; permissions: { file: { public: number; //default 0644 @@ -22,7 +21,6 @@ export class StorageNodeLocalAdapter implements StorageAdapter { protected root: string; protected options: StorageLocalAdapterOptions = { root: '/', - url: '/', permissions: { file: { public: 0o644, @@ -44,6 +42,10 @@ export class StorageNodeLocalAdapter implements StorageAdapter { return true; } + supportsDirectory() { + return true; + } + protected async getFs(): Promise { if (!this.fs) this.fs = await import('fs/promises'); return this.fs; @@ -59,10 +61,6 @@ export class StorageNodeLocalAdapter implements StorageAdapter { return 'private'; } - async url(path: string): Promise { - return resolveStoragePath([this.options.url || this.options.root, path]); - } - getMode(type: FileType, visibility: FileVisibility): number { const permissions = this.options.permissions[type === FileType.File ? 'file' : 'directory']; return visibility === 'public' ? permissions.public : permissions.private; @@ -222,6 +220,11 @@ export class StorageNodeLocalAdapter implements StorageAdapter { const fs = await this.getFs(); await fs.appendFile(path, contents); } + + async setVisibility(path: string, visibility: FileVisibility): Promise { + const fs = await this.getFs(); + await fs.chmod(this.getPath(path), this.getMode(FileType.File, visibility)); + } } export const StorageLocalAdapter = StorageNodeLocalAdapter; diff --git a/packages/storage/src/memory-adapter.ts b/packages/storage/src/memory-adapter.ts index 3b33d393e..c746cb9e3 100644 --- a/packages/storage/src/memory-adapter.ts +++ b/packages/storage/src/memory-adapter.ts @@ -22,12 +22,16 @@ export class StorageMemoryAdapter implements StorageAdapter { return true; } + supportsDirectory() { + return true; + } + async files(path: string): Promise { return this.memory.filter(file => file.file.directory === path) .map(v => v.file); } - async url(path: string): Promise { + async publicUrl(path: string): Promise { return resolveStoragePath([this.options.url, path]); } @@ -127,4 +131,10 @@ export class StorageMemoryAdapter implements StorageAdapter { reporter.progress(++i, files.length); } } + + async setVisibility(path: string, visibility: FileVisibility): Promise { + const file = this.memory.find(file => file.file.path === path); + if (!file) return; + file.file.visibility = visibility; + } } diff --git a/packages/storage/src/storage.ts b/packages/storage/src/storage.ts index 685c1dc7e..dc8c75825 100644 --- a/packages/storage/src/storage.ts +++ b/packages/storage/src/storage.ts @@ -19,6 +19,16 @@ export type FileVisibility = 'public' | 'private'; export class StorageFile { public size: number = 0; public lastModified?: Date; + + /** + * Visibility of the file. + * + * Note that some adapters might not support reading the visibility of a file. + * In this case, the visibility is always 'private'. + * + * Some adapters might support reading the visibility per file, but not when listing files. + * In this case you have to call additional `storage.get(file)` to load the visibility. + */ public visibility: FileVisibility = 'public'; constructor( @@ -109,6 +119,9 @@ export interface Operation extends Promise { export interface StorageAdapter { supportsVisibility(): boolean; + supportsDirectory(): boolean; + + setVisibility?(path: string, visibility: FileVisibility): Promise; /** * Closes the adapter (close connections, etc). @@ -146,12 +159,12 @@ export interface StorageAdapter { makeDirectory(path: string, visibility: FileVisibility): Promise; /** - * Returns the URL for the given path. + * Returns the public URL for the given path. * * For local storage it's the configured base URL + the path. * For adapters like S3 it's the public S3 URL to the file. */ - url(path: string): Promise; + publicUrl?(path: string): Promise; /** * Writes the given contents to the given path. @@ -310,6 +323,13 @@ export interface StorageOptions { * Per default replaces all not allowed characters [^a-zA-Z0-9\.\-\_\/]) with a dash. */ pathNormalizer: (path: string) => string; + + baseUrl?: string; + + /** + * Transforms a given path to a public URL. + */ + urlBuilder: (path: string) => string; } export class Storage { @@ -319,9 +339,18 @@ export class Storage { pathNormalizer: (path: string) => { return path.replace(/[^a-zA-Z0-9\.\-\_]/g, '-'); }, + urlBuilder: (path: string) => { + if (this.options.baseUrl) return this.options.baseUrl + path; + return path; + } }; - constructor(public adapter: StorageAdapter) { + constructor( + public adapter: StorageAdapter, + options: Partial = {} + ) { + Object.assign(this.options, options); + if (this.options.baseUrl?.endsWith('/')) this.options.baseUrl = this.options.baseUrl.slice(0, -1); } /** @@ -500,7 +529,7 @@ export class Storage { const file = await this.get(path); const existing = await this.read(path); - return await this.adapter.write(path as string, new Uint8Array([...existing, ...buffer]), file.visibility, reporter); + return await this.adapter.write(path as string, Buffer.concat([existing, buffer]), file.visibility, reporter); }); } @@ -517,7 +546,7 @@ export class Storage { const file = await this.get(path); const existing = await this.read(path); - return await this.adapter.write(path as string, new Uint8Array([...buffer, ...existing]), file.visibility, reporter); + return await this.adapter.write(path as string, Buffer.concat([buffer, existing]), file.visibility, reporter); }); } @@ -652,9 +681,22 @@ export class Storage { return this.adapter.makeDirectory(path, visibility); } - async url(path: StoragePath): Promise { + /** + * Returns the public URL for the given path. + */ + async publicUrl(path: StoragePath): Promise { path = resolveStoragePath(path); - return await this.adapter.url(path); + if (this.adapter.publicUrl) return await this.adapter.publicUrl(path); + + return this.options.urlBuilder(path); + } + + /** + * Sets the visibility of the given path. + */ + async setVisibility(path: StoragePath, visibility: FileVisibility): Promise { + if (!this.adapter.setVisibility) return; + await this.adapter.setVisibility(resolveStoragePath(path), visibility); } } diff --git a/packages/storage/tests/local.spec.ts b/packages/storage/tests/local.spec.ts index 2ef9a05bb..610b14fa3 100644 --- a/packages/storage/tests/local.spec.ts +++ b/packages/storage/tests/local.spec.ts @@ -8,7 +8,7 @@ import { Storage } from '../src/storage.js'; setAdapterFactory(async () => { const tmp = mkdtempSync(tmpdir() + '/storage-test-'); - return new StorageLocalAdapter({root: tmp, url: '/files/'}); + return new StorageLocalAdapter({root: tmp}); }); // since we import .storage.spec.js, all its tests are scheduled to run @@ -20,10 +20,3 @@ test('recursive', () => undefined); test('permissions', () => undefined); test('copy', () => undefined); test('move', () => undefined); - -test('urls', async() => { - const storage = new Storage(await adapterFactory()); - await storage.write('/file1.txt', 'contents1', 'public'); - - expect(await storage.url('/file1.txt')).toBe('/files/file1.txt'); -}); diff --git a/packages/storage/tests/storage.spec.ts b/packages/storage/tests/storage.spec.ts index 87ab5e4d0..61c7ef475 100644 --- a/packages/storage/tests/storage.spec.ts +++ b/packages/storage/tests/storage.spec.ts @@ -8,6 +8,15 @@ export function setAdapterFactory(factory: () => Promise) { adapterFactory = factory; } +test('url', async () => { + const storage = new Storage(await adapterFactory(), { baseUrl: 'http://localhost/assets/' }); + if (storage.adapter.publicUrl) return; //has custom tests + + //this test is about URL mapping feature from Storage + const url = await storage.publicUrl('/file1.txt'); + expect(url).toBe('http://localhost/assets/file1.txt'); +}); + test('basic', async () => { const storage = new Storage(await adapterFactory()); @@ -44,6 +53,9 @@ test('basic', async () => { expect(await storage.exists('/file4.txt')).toBe(false); expect(await storage.exists('//file4.txt')).toBe(false); + await storage.write('/file1.txt', 'overridden'); + expect(await storage.readAsText('/file1.txt')).toBe('overridden'); + await storage.delete('/file1.txt'); await storage.delete('/file2.txt'); @@ -73,7 +85,7 @@ test('append/prepend', async () => { await storage.close(); }); -test('permissions', async () => { +test('visibility', async () => { const adapter = await adapterFactory(); if (!adapter.supportsVisibility()) { if (adapter.close) await adapter.close(); @@ -90,14 +102,20 @@ test('permissions', async () => { const file2 = await storage.get('/file2.txt'); expect(file2).toMatchObject({ path: '/file2.txt', size: 9, lastModified: expect.any(Date), visibility: 'private' }); - await storage.makeDirectory('/folder1', 'public'); - await storage.makeDirectory('/folder2', 'private'); + if (adapter.supportsDirectory()) { + await storage.makeDirectory('/folder1', 'public'); + await storage.makeDirectory('/folder2', 'private'); - const folder1 = await storage.get('/folder1'); - expect(folder1).toMatchObject({ path: '/folder1', size: 0, visibility: 'public' }); + const folder1 = await storage.get('/folder1'); + expect(folder1).toMatchObject({ path: '/folder1', size: 0, visibility: 'public' }); - const folder2 = await storage.get('/folder2'); - expect(folder2).toMatchObject({ path: '/folder2', size: 0, visibility: 'private' }); + const folder2 = await storage.get('/folder2'); + expect(folder2).toMatchObject({ path: '/folder2', size: 0, visibility: 'private' }); + + } + await storage.setVisibility('file2.txt', 'public'); + const file2b = await storage.get('/file2.txt'); + expect(file2b).toMatchObject({ path: '/file2.txt', size: 9, lastModified: expect.any(Date), visibility: 'public' }); await storage.close(); }); @@ -128,7 +146,8 @@ test('recursive', async () => { const files3 = await storage.allFiles('/'); const fileNames3 = files3.map(f => f.path); - expect(fileNames3).toEqual([ + + let expected = [ '/folder', '/folder2', '/folder2/folder3', @@ -138,7 +157,12 @@ test('recursive', async () => { '/folder2/file2.txt', '/folder2/file3.txt', '/folder2/folder3/file4.txt', - ]); + ]; + + if (!storage.adapter.supportsDirectory()) { + expected = expected.filter(v => v !== '/folder' && v !== '/folder2' && v !== '/folder2/folder3'); + } + expect(fileNames3).toEqual(expected); const directories = await storage.directories('/'); expect(directories).toMatchObject([ @@ -151,12 +175,14 @@ test('recursive', async () => { { path: '/folder2/folder3', type: FileType.Directory }, ]); - const directories3 = await storage.allDirectories('/'); - expect(directories3).toMatchObject([ - { path: '/folder', type: FileType.Directory }, - { path: '/folder2', type: FileType.Directory }, - { path: '/folder2/folder3', type: FileType.Directory }, - ]); + if (storage.adapter.supportsDirectory()) { + const directories3 = await storage.allDirectories('/'); + expect(directories3).toMatchObject([ + { path: '/folder', type: FileType.Directory }, + { path: '/folder2', type: FileType.Directory }, + { path: '/folder2/folder3', type: FileType.Directory }, + ]); + } await storage.close(); });