feature(storage): add ftp adapter

.github/workflows/main.yml

@@ -10,6 +10,15 @@ jobs:
     strategy:
       matrix:
         node-version: [18.17.0]
+    services:
+        storage-ftp:
+            image: garethflowers/ftp-server:0.7.0
+            env:
+                FTP_USER: user
+                FTP_PASS: '123'
+            ports:
+                - "20-21:20-21"
+                - "40000-40009:40000-40009"
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}

packages/storage-aws-s3/src/s3-adapter.ts

@@ -41,6 +41,10 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         });
     }
 
+    supportsVisibility() {
+        return true;
+    }
+
     protected isDirectorySupport(): boolean {
         return this.options.directorySupport !== false;
     }
@@ -65,6 +69,10 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         return path.slice(base.length);
     }
 
+    async url(path: string): Promise<string> {
+        return `s3://${this.options.bucket}/${this.getRemotePath(path)}`;
+    }
+
     async makeDirectory(path: string): Promise<void> {
         if (path === '') return;
 
@@ -119,7 +127,7 @@ export class StorageAwsS3Adapter implements StorageAdapter {
                     if (content.Key === remotePath) continue;
 
                     const file = new StorageFile(this.pathMapToVirtual(content.Key));
-                    file.size = content.Size;
+                    file.size = content.Size || 0;
                     file.lastModified = content.LastModified;
                     file.type = content.Key.endsWith('/') ? FileType.Directory : FileType.File;
                     files.push(file);
@@ -242,7 +250,7 @@ export class StorageAwsS3Adapter implements StorageAdapter {
         });
         try {
             const response = await this.client.send(command);
-            file.size = response.ContentLength;
+            file.size = response.ContentLength || 0;
             file.lastModified = response.LastModified;
         } catch (error: any) {
             return undefined;

packages/storage-ftp/.npmignore

@@ -0,0 +1 @@
+tests

packages/storage-ftp/README.md

@@ -0,0 +1,14 @@
+# skeleton package
+
+This package can be copied when a new package should be created.
+
+### Steps after copying:
+
+- Adjust "name", "description", and "private" in `package.json`.
+- Adjust README.md
+- Put this package into root `/package.json` "jest.references".
+- Put this package into root `/tsconfig.json` "references".
+- Put this package into root `/tsconfig.esm.json` "references".
+- Add dependencies to `package.json` and run `node sync-tsconfig-deps.js` to adjust tsconfig automatically.
+- Add to .github/workflows/main.yml tsc build step if necessary.
+- Add to typedoc build in deepkit-website if necessary.

packages/storage-ftp/index.ts

@@ -0,0 +1 @@
+export * from './src/ftp-adapter.js';

packages/storage-ftp/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@deepkit/storage-ftp",
+  "version": "1.0.1-alpha.13",
+  "description": "Deepkit storage adapter for FTP",
+  "private": true,
+  "type": "commonjs",
+  "main": "./dist/cjs/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/cjs/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/cjs/index.d.ts",
+      "require": "./dist/cjs/index.js",
+      "default": "./dist/esm/index.js"
+    }
+  },
+  "repository": "https://github.com/deepkit/deepkit-framework",
+  "author": "Marc J. Schmidt <[email protected]>",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json"
+  },
+  "dependencies": {
+    "basic-ftp": "^5.0.3"
+  },
+  "devDependencies": {
+    "@deepkit/storage": "^1.0.1-alpha.13"
+  },
+  "jest": {
+    "runner": "../../jest-serial-runner.js",
+    "testEnvironment": "node",
+    "transform": {
+      "^.+\\.(ts|tsx)$": [
+        "ts-jest",
+        {
+          "tsconfig": "<rootDir>/tsconfig.json"
+        }
+      ]
+    },
+    "moduleNameMapper": {
+      "(.+)\\.js": "$1"
+    },
+    "testMatch": [
+      "**/tests/**/*.spec.ts"
+    ]
+  }
+}

packages/storage-ftp/src/ftp-adapter.ts

@@ -0,0 +1,222 @@
+import { FileType, FileVisibility, pathBasename, pathDirectory, Reporter, resolveStoragePath, StorageAdapter, StorageFile } from '@deepkit/storage';
+import { Client, FileInfo } from 'basic-ftp';
+import type { ConnectionOptions as TLSConnectionOptions } from 'tls';
+import { Readable, Writable } from 'stream';
+
+export interface StorageFtpOptions {
+    /**
+     * The root path where all files are stored. Optional, default is )" (standard working directory of FTP server_.
+     */
+    root: string;
+
+    /**
+     * Host the client should connect to. Optional, default is "localhost".
+     */
+    host: string;
+
+    /**
+     * Port the client should connect to. Optional, default is 21.
+     */
+    port?: number;
+
+    /**
+     * Timeout in secnds for all client commands. Optional, default is 30 seconds.
+     */
+    timeout?: number;
+
+    user: string;
+
+    password: string;
+
+    /**
+     * Use FTPS over TLS. Optional, default is false.
+     * True is preferred explicit TLS, "implicit" supports legacy, non-standardized implicit TLS.
+     */
+    secure?: boolean;
+    secureOptions?: TLSConnectionOptions;
+}
+
+export class StorageFtpAdapter implements StorageAdapter {
+    client: Client;
+    options: StorageFtpOptions = {
+        root: '',
+        host: 'localhost',
+        user: '',
+        password: '',
+    };
+
+    constructor(options: Partial<StorageFtpOptions> = {}) {
+        Object.assign(this.options, options);
+        this.client = new Client(this.options.timeout);
+    }
+
+    supportsVisibility() {
+        return false;
+    }
+
+    async clearWorkingDir() {
+        await this.ensureConnected();
+        await this.client.clearWorkingDir();
+    }
+
+    protected getRemotePath(path: string): string {
+        if (this.options.root === '') return path;
+        return resolveStoragePath([this.options.root, path]);
+    }
+
+    async url(path: string): Promise<string> {
+        return `ftp://${this.options.host}:${this.options.port}/${this.getRemotePath(path)}`;
+    }
+
+    async close(): Promise<void> {
+        this.client.close();
+    }
+
+    async ensureConnected(): Promise<void> {
+        if (!this.client.closed) return;
+        await this.client.access({
+            host: this.options.host,
+            port: this.options.port,
+            user: this.options.user,
+            password: this.options.password,
+            secure: this.options.secure,
+            secureOptions: this.options.secureOptions,
+        });
+    }
+
+    async makeDirectory(path: string, visibility: FileVisibility): Promise<void> {
+        await this.ensureConnected();
+        const remotePath = this.getRemotePath(path);
+        await this.client.ensureDir(remotePath);
+    }
+
+    async files(path: string): Promise<StorageFile[]> {
+        return await this.getFiles(path, false);
+    }
+
+    protected async getFiles(path: string, recursive: boolean = false): Promise<StorageFile[]> {
+        await this.ensureConnected();
+        const remotePath = this.getRemotePath(path);
+        const entries = await this.client.list(remotePath);
+        return entries.map(v => this.createStorageFile(path + '/' + v.name, v));
+    }
+
+    async delete(paths: string[]): Promise<void> {
+        await this.ensureConnected();
+        for (const path of paths) {
+            const remotePath = this.getRemotePath(path);
+            await this.client.remove(remotePath);
+        }
+    }
+
+    async deleteDirectory(path: string, reporter: Reporter): Promise<void> {
+        await this.ensureConnected();
+        const remotePath = this.getRemotePath(path);
+        await this.client.removeDir(remotePath);
+    }
+
+    async exists(paths: string[]): Promise<boolean> {
+        await this.ensureConnected();
+        const foldersToCheck: { folder: string, names: string[] }[] = [];
+
+        for (const path of paths) {
+            if (path === '/') continue;
+            const folder = pathDirectory(path);
+            const entry = foldersToCheck.find(v => v.folder === folder);
+            if (entry) {
+                entry.names.push(pathBasename(path));
+            } else {
+                foldersToCheck.push({ folder, names: [pathBasename(path)] });
+            }
+        }
+
+        for (const folders of foldersToCheck) {
+            const remotePath = this.getRemotePath(folders.folder);
+            const files = await this.client.list(remotePath);
+            for (const name of folders.names) {
+                if (!files.find(v => v.name === name)) return false;
+            }
+        }
+
+        return true;
+    }
+
+    async get(path: string): Promise<StorageFile | undefined> {
+        if (path === '/') return;
+        await this.ensureConnected();
+        const remotePath = this.getRemotePath(pathDirectory(path));
+        const files = await this.client.list(remotePath);
+        const basename = pathBasename(path);
+        const entry = files.find(v => v.name === basename);
+        if (!entry) return;
+        return this.createStorageFile(path, entry);
+    }
+
+    protected createStorageFile(path: string, fileInfo: FileInfo): StorageFile {
+        const file = new StorageFile(path, fileInfo.isFile ? FileType.File : FileType.Directory);
+        file.size = fileInfo.size;
+        file.lastModified = fileInfo.modifiedAt;
+        if (!file.lastModified && fileInfo.rawModifiedAt) {
+            file.lastModified = parseCustomDateString(fileInfo.rawModifiedAt);
+        }
+        return file;
+    }
+
+    async move(source: string, destination: string, reporter: Reporter): Promise<void> {
+        await this.client.rename(this.getRemotePath(source), this.getRemotePath(destination));
+    }
+
+    async read(path: string, reporter: Reporter): Promise<Uint8Array> {
+        await this.ensureConnected();
+        const remotePath = this.getRemotePath(path);
+        const chunks: Uint8Array[] = [];
+        const writeable = new Writable({
+            write(chunk: any, encoding: BufferEncoding, callback: (error?: (Error | null)) => void) {
+                chunks.push(chunk);
+                callback(null);
+            }
+        });
+        const stream = await this.client.downloadTo(writeable, remotePath);
+        return Buffer.concat(chunks);
+    }
+
+    async write(path: string, contents: Uint8Array, visibility: FileVisibility, reporter: Reporter): Promise<void> {
+        await this.ensureConnected();
+        await this.client.ensureDir(this.getRemotePath(pathDirectory(path)));
+        await this.client.uploadFrom(createReadable(contents), this.getRemotePath(path));
+    }
+}
+
+/**
+ * Best effort to parse date strings like `22 Oct 10 12:45` or `Oct 10 12:45` into a Date object.
+ */
+function parseCustomDateString(dateString: string): Date | undefined {
+    const currentYear = new Date().getFullYear();
+
+    const twoDigitYearMatch = dateString.match(/^\d{2}\s/);
+    const fourDigitYearMatch = dateString.match(/^\d{4}\s/);
+
+    let fullDateString;
+
+    if (twoDigitYearMatch) {
+        // Handle '22 Oct 10 12:45' format.
+        const twoDigitYear = twoDigitYearMatch[0].trim();
+        const baseYear = currentYear.toString().substring(0, 2); // Get the first two digits of the current year.
+        fullDateString = `${baseYear}${twoDigitYear} ${dateString.substring(3)}`;
+    } else if (fourDigitYearMatch) {
+        // Handle '2022 Oct 10 12:45' format.
+        fullDateString = dateString;
+    } else {
+        // Handle 'Oct 10 12:45' format.
+        fullDateString = `${dateString} ${currentYear}`;
+    }
+
+    return new Date(fullDateString);
+}
+
+function createReadable(buffer: Uint8Array): Readable {
+    const stream = new Readable();
+    stream.push(buffer);
+    stream.push(null);
+    return stream;
+}

packages/storage-ftp/tests/storage.spec.ts

@@ -0,0 +1,35 @@
+import { test } from '@jest/globals';
+import './storage.spec.js';
+import { setAdapterFactory } from '@deepkit/storage/test';
+import { StorageFtpAdapter } from '../src/ftp-adapter.js';
+import { platform } from 'os';
+
+setAdapterFactory(async () => {
+    let adapter = new StorageFtpAdapter({
+        host: 'localhost',
+        user: 'user',
+        password: '123',
+    });;
+    if (platform() === 'darwin') {
+        // docker run -d --name storage-ftp -p 20-21:20-21 -p 40000-40009:40000-40009 --env FTP_USER=user --env FTP_PASS=123 garethflowers/ftp-server
+        adapter = new StorageFtpAdapter({
+            host: 'storage-ftp.orb.local',
+            port: 21,
+            user: 'user',
+            password: '123',
+        });
+    }
+    //reset all files
+    await adapter.clearWorkingDir();
+
+    return adapter;
+});
+
+// since we import .storage.spec.js, all its tests are scheduled to run
+// we define 'basic' here too, so we can easily run just this test.
+// also necessary to have at least once test in this file, so that WebStorm
+// detects the file as a test file.
+test('basic', () => undefined);
+test('recursive', () => undefined);
+test('copy', () => undefined);
+test('move', () => undefined);