feature(storage): add publicUrl support

packages/storage-aws-s3/README.md

@@ -1,14 +1,5 @@
-# skeleton package
+# Deepkit Storage AWS S3
 
-This package can be copied when a new package should be created.
+This package provides Deepkit Storage integration for AWS S3.
 
-### Steps after copying:
-
-- Adjust "name", "description", and "private" in `package.json`.
-- Adjust README.md
-- Put this package into root `/package.json` "jest.references".
-- Put this package into root `/tsconfig.json` "references".
-- Put this package into root `/tsconfig.esm.json` "references".
-- Add dependencies to `package.json` and run `node sync-tsconfig-deps.js` to adjust tsconfig automatically.
-- Add to .github/workflows/main.yml tsc build step if necessary.
-- Add to typedoc build in deepkit-website if necessary.
+If you create a S3 bucket and want (public) visibility support, make sure ACLs is enabled and "Block public access" is disabled.

packages/storage-aws-s3/src/s3-adapter.ts

@@ -2,12 +2,13 @@ import { FileType, FileVisibility, pathDirectory, Reporter, StorageAdapter, Stor
 import {
     CopyObjectCommand,
     DeleteObjectsCommand,
+    GetObjectAclCommand,
     GetObjectCommand,
     HeadObjectCommand,
     ListObjectsCommand,
-    ListObjectsV2Command,
+    PutObjectAclCommand,
     PutObjectCommand,
-    S3Client
+    S3Client,
 } from '@aws-sdk/client-s3';
 import { normalizePath } from 'typedoc';
 
@@ -24,7 +25,7 @@ export interface StorageAwsS3Options {
      *
      * Default: true
      */
-    directorySupport?: boolean;
+    directoryEmulation?: boolean;
 }
 
 export class StorageAwsS3Adapter implements StorageAdapter {
@@ -45,8 +46,8 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         return true;
     }
 
-    protected isDirectorySupport(): boolean {
-        return this.options.directorySupport !== false;
+    supportsDirectory() {
+        return this.options.directoryEmulation === true;
     }
 
     protected getRemotePath(path: string) {
@@ -69,17 +70,33 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         return path.slice(base.length);
     }
 
-    async url(path: string): Promise<string> {
-        return `s3://${this.options.bucket}/${this.getRemotePath(path)}`;
+    async publicUrl(path: string): Promise<string> {
+        return `https://${this.options.bucket}.s3.${this.options.region}.amazonaws.com/${this.getRemotePath(path)}`;
     }
 
-    async makeDirectory(path: string): Promise<void> {
+    protected visibilityToAcl(visibility: FileVisibility): string {
+        if (visibility === 'public') return 'public-read';
+        return 'private';
+    }
+
+    protected aclToVisibility(grants: any[]): FileVisibility {
+        for (const grant of grants) {
+            if (grant.Permission === 'READ' && grant.Grantee.URI === 'http://acs.amazonaws.com/groups/global/AllUsers') {
+                return 'public';
+            }
+        }
+
+        return 'private';
+    }
+
+    async makeDirectory(path: string, visibility: FileVisibility): Promise<void> {
         if (path === '') return;
 
         const command = new PutObjectCommand({
             Bucket: this.options.bucket,
             Key: this.getRemoteDirectory(path),
-            ContentLength: 0
+            ContentLength: 0,
+            ACL: this.visibilityToAcl(visibility),
         });
 
         try {
@@ -101,8 +118,7 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         const files: StorageFile[] = [];
         const remotePath = this.getRemoteDirectory(path);
 
-        //only v2 includes directories
-        const command = new ListObjectsV2Command({
+        const command = new ListObjectsCommand({
             Bucket: this.options.bucket,
             Prefix: remotePath,
             Delimiter: recursive ? undefined : '/',
@@ -144,8 +160,8 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         const file = await this.get(source);
 
         if (file && file.isFile()) {
-            if (this.isDirectorySupport()) {
-                await this.makeDirectory(pathDirectory(destination));
+            if (this.supportsDirectory()) {
+                await this.makeDirectory(pathDirectory(destination), file.visibility);
             }
 
             const command = new CopyObjectCommand({
@@ -252,6 +268,7 @@ export class StorageAwsS3Adapter implements StorageAdapter {
             const response = await this.client.send(command);
             file.size = response.ContentLength || 0;
             file.lastModified = response.LastModified;
+            file.visibility = await this.getVisibility(path);
         } catch (error: any) {
             return undefined;
         }
@@ -283,15 +300,16 @@ export class StorageAwsS3Adapter implements StorageAdapter {
     }
 
     async write(path: string, contents: Uint8Array, visibility: FileVisibility, reporter: Reporter): Promise<void> {
-        if (this.isDirectorySupport()) {
-            await this.makeDirectory(pathDirectory(path));
+        if (this.supportsDirectory()) {
+            await this.makeDirectory(pathDirectory(path), visibility);
         }
 
         const remotePath = this.getRemotePath(path);
         const command = new PutObjectCommand({
             Bucket: this.options.bucket,
             Key: remotePath,
             Body: contents,
+            ACL: this.visibilityToAcl(visibility),
         });
 
         try {
@@ -300,4 +318,34 @@ export class StorageAwsS3Adapter implements StorageAdapter {
             throw new StorageError(`Could not write file ${path}: ${error.message}`);
         }
     }
+
+    async getVisibility(path: string): Promise<FileVisibility> {
+        const remotePath = this.getRemotePath(path);
+        const aclCommand = new GetObjectAclCommand({
+            Bucket: this.options.bucket,
+            Key: remotePath,
+        });
+
+        try {
+            const response = await this.client.send(aclCommand);
+            return this.aclToVisibility(response.Grants || []);
+        } catch (error: any) {
+            throw new StorageError(`Could not get visibility for ${path}: ${error.message}`);
+        }
+    }
+
+    async setVisibility(path: string, visibility: FileVisibility): Promise<void> {
+        const remotePath = this.getRemotePath(path);
+        const command = new PutObjectAclCommand({
+            Bucket: this.options.bucket,
+            Key: remotePath,
+            ACL: this.visibilityToAcl(visibility),
+        });
+
+        try {
+            await this.client.send(command);
+        } catch (error: any) {
+            throw new StorageError(`Could not set visibility for ${path}: ${error.message}`);
+        }
+    }
 }

packages/storage-aws-s3/tests/storage.spec.ts

@@ -1,8 +1,9 @@
-import { test } from '@jest/globals';
+import { expect, test } from '@jest/globals';
 import './storage.spec.js';
-import { setAdapterFactory } from '@deepkit/storage/test';
+import { adapterFactory, setAdapterFactory } from '@deepkit/storage/test';
 import { StorageAwsS3Adapter } from '../src/s3-adapter.js';
 import { DeleteObjectsCommand, ListObjectsCommand } from '@aws-sdk/client-s3';
+import { Storage } from '@deepkit/storage';
 
 setAdapterFactory(async () => {
     const folder = 'test-folder-dont-delete';
@@ -38,11 +39,20 @@ setAdapterFactory(async () => {
     return adapter;
 });
 
+test('s3 url', async () => {
+    const storage = new Storage(await adapterFactory());
+
+    await storage.write('test.txt', 'abc', 'public');
+    const url = await storage.publicUrl('test.txt');
+    expect(url).toBe('https://deepkit-storage-integration-tests.s3.eu-central-1.amazonaws.com/test-folder-dont-delete/test.txt');
+});
+
 // since we import .storage.spec.js, all its tests are scheduled to run
 // we define 'basic' here too, so we can easily run just this test.
 // also necessary to have at least once test in this file, so that WebStorm
 // detects the file as a test file.
 test('basic', () => undefined);
+test('visibility', () => undefined);
 test('recursive', () => undefined);
 test('copy', () => undefined);
 test('move', () => undefined);

packages/storage-ftp/src/ftp-adapter.ts

@@ -54,6 +54,10 @@ export class StorageFtpAdapter implements StorageAdapter {
         return false;
     }
 
+    supportsDirectory() {
+        return true;
+    }
+
     async clearWorkingDir() {
         await this.ensureConnected();
         await this.client.clearWorkingDir();
@@ -64,7 +68,7 @@ export class StorageFtpAdapter implements StorageAdapter {
         return resolveStoragePath([this.options.root, path]);
     }
 
-    async url(path: string): Promise<string> {
+    async publicUrl(path: string): Promise<string> {
         return `ftp://${this.options.host}:${this.options.port}/${this.getRemotePath(path)}`;
     }
 

packages/storage/src/local-adapter.ts

@@ -3,7 +3,6 @@ import type * as fs from 'fs/promises';
 
 export interface StorageLocalAdapterOptions {
     root: string;
-    url: string;
     permissions: {
         file: {
             public: number; //default 0644
@@ -22,7 +21,6 @@ export class StorageNodeLocalAdapter implements StorageAdapter {
     protected root: string;
     protected options: StorageLocalAdapterOptions = {
         root: '/',
-        url: '/',
         permissions: {
             file: {
                 public: 0o644,
@@ -44,6 +42,10 @@ export class StorageNodeLocalAdapter implements StorageAdapter {
         return true;
     }
 
+    supportsDirectory() {
+        return true;
+    }
+
     protected async getFs(): Promise<typeof fs> {
         if (!this.fs) this.fs = await import('fs/promises');
         return this.fs;
@@ -59,10 +61,6 @@ export class StorageNodeLocalAdapter implements StorageAdapter {
         return 'private';
     }
 
-    async url(path: string): Promise<string> {
-        return resolveStoragePath([this.options.url || this.options.root, path]);
-    }
-
     getMode(type: FileType, visibility: FileVisibility): number {
         const permissions = this.options.permissions[type === FileType.File ? 'file' : 'directory'];
         return visibility === 'public' ? permissions.public : permissions.private;
@@ -222,6 +220,11 @@ export class StorageNodeLocalAdapter implements StorageAdapter {
         const fs = await this.getFs();
         await fs.appendFile(path, contents);
     }
+
+    async setVisibility(path: string, visibility: FileVisibility): Promise<void> {
+        const fs = await this.getFs();
+        await fs.chmod(this.getPath(path), this.getMode(FileType.File, visibility));
+    }
 }
 
 export const StorageLocalAdapter = StorageNodeLocalAdapter;

packages/storage/src/memory-adapter.ts

@@ -22,12 +22,16 @@ export class StorageMemoryAdapter implements StorageAdapter {
         return true;
     }
 
+    supportsDirectory() {
+        return true;
+    }
+
     async files(path: string): Promise<StorageFile[]> {
         return this.memory.filter(file => file.file.directory === path)
             .map(v => v.file);
     }
 
-    async url(path: string): Promise<string> {
+    async publicUrl(path: string): Promise<string> {
         return resolveStoragePath([this.options.url, path]);
     }
 
@@ -127,4 +131,10 @@ export class StorageMemoryAdapter implements StorageAdapter {
             reporter.progress(++i, files.length);
         }
     }
+
+    async setVisibility(path: string, visibility: FileVisibility): Promise<void> {
+        const file = this.memory.find(file => file.file.path === path);
+        if (!file) return;
+        file.file.visibility = visibility;
+    }
 }