diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5596653f4..208de5622 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,11 @@ jobs: ports: - "20-21:20-21" - "40000-40009:40000-40009" + storage-sftp: + image: atmoz/sftp:alpine + ports: + - "22:22" + options: user:123:::upload steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} diff --git a/packages/storage-ftp/src/ftp-adapter.ts b/packages/storage-ftp/src/ftp-adapter.ts index 520707898..15555fb00 100644 --- a/packages/storage-ftp/src/ftp-adapter.ts +++ b/packages/storage-ftp/src/ftp-adapter.ts @@ -275,7 +275,6 @@ export class StorageFtpAdapter implements StorageAdapter { await this.chmodRecursive(path, permission); } - } /** diff --git a/packages/storage-ftp/tests/storage.spec.ts b/packages/storage-ftp/tests/storage.spec.ts index 21f7f312f..6dc87d8ea 100644 --- a/packages/storage-ftp/tests/storage.spec.ts +++ b/packages/storage-ftp/tests/storage.spec.ts @@ -9,7 +9,8 @@ setAdapterFactory(async () => { host: 'localhost', user: 'user', password: '123', - });; + }); + if (platform() === 'darwin') { // docker run -d --name storage-ftp -p 20-21:20-21 -p 40000-40009:40000-40009 --env FTP_USER=user --env FTP_PASS=123 garethflowers/ftp-server adapter = new StorageFtpAdapter({ diff --git a/packages/storage-sftp/.npmignore b/packages/storage-sftp/.npmignore new file mode 100644 index 000000000..2b29f2764 --- /dev/null +++ b/packages/storage-sftp/.npmignore @@ -0,0 +1 @@ +tests diff --git a/packages/storage-sftp/README.md b/packages/storage-sftp/README.md new file mode 100644 index 000000000..a41b39584 --- /dev/null +++ b/packages/storage-sftp/README.md @@ -0,0 +1 @@ +# Deepkit SFTP (SSH) Storage adapter diff --git a/packages/storage-sftp/dist/.gitkeep b/packages/storage-sftp/dist/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/packages/storage-sftp/index.ts b/packages/storage-sftp/index.ts new file mode 100644 index 000000000..46f41587f --- /dev/null +++ b/packages/storage-sftp/index.ts @@ -0,0 +1 @@ +export * from './src/ftp-adapter.js'; diff --git a/packages/storage-sftp/package.json b/packages/storage-sftp/package.json new file mode 100644 index 000000000..85b892d47 --- /dev/null +++ b/packages/storage-sftp/package.json @@ -0,0 +1,51 @@ +{ + "name": "@deepkit/storage-fstp", + "version": "1.0.1-alpha.13", + "description": "Deepkit storage adapter for sFTP (via SSH)", + "private": true, + "type": "commonjs", + "main": "./dist/cjs/index.js", + "module": "./dist/esm/index.js", + "types": "./dist/cjs/index.d.ts", + "exports": { + ".": { + "types": "./dist/cjs/index.d.ts", + "require": "./dist/cjs/index.js", + "default": "./dist/esm/index.js" + } + }, + "repository": "https://github.com/deepkit/deepkit-framework", + "author": "Marc J. Schmidt ", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json" + }, + "dependencies": { + "ssh2-sftp-client": "^9.1.0" + }, + "devDependencies": { + "@deepkit/storage": "^1.0.1-alpha.13", + "@types/ssh2-sftp-client": "^9.0.1" + }, + "jest": { + "runner": "../../jest-serial-runner.js", + "testEnvironment": "node", + "transform": { + "^.+\\.(ts|tsx)$": [ + "ts-jest", + { + "tsconfig": "/tsconfig.json" + } + ] + }, + "moduleNameMapper": { + "(.+)\\.js": "$1" + }, + "testMatch": [ + "**/tests/**/*.spec.ts" + ] + } +} diff --git a/packages/storage-sftp/src/sftp-adapter.ts b/packages/storage-sftp/src/sftp-adapter.ts new file mode 100644 index 000000000..e55eb386f --- /dev/null +++ b/packages/storage-sftp/src/sftp-adapter.ts @@ -0,0 +1,314 @@ +import { FileType, FileVisibility, pathBasename, pathDirectory, Reporter, resolveStoragePath, StorageAdapter, StorageFile } from '@deepkit/storage'; +import Client, { ConnectOptions, FileInfo } from 'ssh2-sftp-client'; +import { Readable } from 'stream'; +import { asyncOperation } from '@deepkit/core'; + +export interface StorageFtpOptions { + /** + * The root path where all files are stored. Optional, default is )" (standard working directory of FTP server_. + */ + root: string; + + /** + * Host the client should connect to. Optional, default is "localhost". + */ + host: string; + + /** + * Port the client should connect to. Optional, default is 21. + */ + port?: number; + + /** + * Timeout in secnds for all client commands. Optional, default is 30 seconds. + */ + timeout?: number; + + user: string; + + password: string; + + options?: ConnectOptions; + + permissions: { + file: { + public: number; //default 0o644 + private: number; //default 0o600 + }, + directory: { + public: number; //default 0o755 + private: number; //default 0o700 + } + }; +} + +/** + * String mode is 'r', 'w', 'rw', 'rwx', etc + * convert to octal representation, rw = 0o6, rwx = 0o7, etc + */ +function stringModeToNumber(mode: string): number { + let result = 0; + if (mode.includes('r')) result += 4; + if (mode.includes('w')) result += 2; + if (mode.includes('x')) result += 1; + return result; +} + +export class StorageSftpAdapter implements StorageAdapter { + client: Client; + options: StorageFtpOptions = { + root: '', + host: 'localhost', + user: '', + password: '', + permissions: { + file: { + public: 0o644, + private: 0o600 + }, + directory: { + public: 0o755, + private: 0o700 + } + } + }; + protected closed = true; + + protected connectPromise?: Promise; + + constructor(options: Partial = {}) { + Object.assign(this.options, options); + if (options.options) Object.assign(this.options, options.options); + this.client = new Client(); + this.client.on('end', (err) => { + this.closed = true; + }); + } + + supportsVisibility() { + return true; + } + + supportsDirectory() { + return true; + } + + protected getRemotePath(path: string): string { + if (this.options.root === '') return path; + return resolveStoragePath([this.options.root, path]); + } + + protected stringModeToMode(rights: { user: string, group: string, other: string }): number { + const user = stringModeToNumber(rights.user); + const group = stringModeToNumber(rights.group); + const other = stringModeToNumber(rights.other); + return user * 64 + group * 8 + other; + } + + protected mapModeToVisibility(type: FileType, rights: { user: string, group: string, other: string }): FileVisibility { + const permissions = this.options.permissions[type === FileType.File ? 'file' : 'directory']; + // example rights.user="rwx", rights.group="rwx", rights.other="rwx" + const mode = this.stringModeToMode(rights); + if (mode === permissions.public) return 'public'; + return 'private'; + } + + async getVisibility(path: string): Promise { + await this.ensureConnected(); + const file = await this.get(path); + if (!file) throw new Error(`File ${path} not found`); + return file.visibility; + } + + async setVisibility(path: string, visibility: FileVisibility): Promise { + await this.ensureConnected(); + await this.chmod(path, this.getMode(FileType.File, visibility)); + } + + protected async chmodFile(path: string, permission: number) { + await this.client.chmod(this.getRemotePath(path), permission); + } + + protected async chmodRecursive(path: string, permission: number) { + const dirs: string[] = [path]; + + while (dirs.length > 0) { + const dir = dirs.pop()!; + const files = await this.client.list(this.getRemotePath(dir)); + for (const file of files) { + const path = dir + '/' + file.name; + if (file.type === 'd') { + dirs.push(path); + } else { + await this.chmodFile(path, permission); + } + } + } + } + + protected async chmod(path: string, permission: number) { + const file = await this.get(path); + if (!file) return; + if (file.isFile()) { + await this.chmodFile(path, permission); + return; + } + + await this.chmodRecursive(path, permission); + } + + protected getMode(type: FileType, visibility: FileVisibility): number { + const permissions = this.options.permissions[type === FileType.File ? 'file' : 'directory']; + return visibility === 'public' ? permissions.public : permissions.private; + } + + async publicUrl(path: string): Promise { + return `sftp://${this.options.host}:${this.options.port}/${this.getRemotePath(path)}`; + } + + async close(): Promise { + await this.client.end(); + } + + async ensureConnected(): Promise { + if (this.connectPromise) await this.connectPromise; + if (!this.closed) return; + this.connectPromise = asyncOperation(async (resolve) => { + this.closed = false; + await this.client.connect({ + host: this.options.host, + port: this.options.port, + username: this.options.user, + password: this.options.password, + }); + this.connectPromise = undefined; + resolve(undefined); + }); + await this.connectPromise; + } + + async makeDirectory(path: string, visibility: FileVisibility): Promise { + await this.ensureConnected(); + if (path === '/') return; + await this.client.mkdir(this.getRemotePath(path), true); + await this.client.chmod(this.getRemotePath(path), this.getMode(FileType.Directory, visibility)); + } + + async files(path: string): Promise { + return await this.getFiles(path, false); + } + + protected async getFiles(path: string, recursive: boolean = false): Promise { + await this.ensureConnected(); + const remotePath = this.getRemotePath(path); + const entries = await this.client.list(remotePath); + return entries.map(v => this.createStorageFile(path + '/' + v.name, v)); + } + + async delete(paths: string[]): Promise { + await this.ensureConnected(); + for (const path of paths) { + const file = await this.get(path); + if (!file) continue; + if (file?.isDirectory()) { + await this.client.rmdir(this.getRemotePath(path), true); + } else { + await this.client.delete(this.getRemotePath(path)); + } + } + } + + async deleteDirectory(path: string, reporter: Reporter): Promise { + await this.ensureConnected(); + if (path === '/') { + for (const file of await this.client.list(this.getRemotePath(path))) { + await this.client.delete(this.getRemotePath(file.name)); + } + } else { + await this.client.rmdir(this.getRemotePath(path), true); + } + } + + async exists(paths: string[]): Promise { + await this.ensureConnected(); + + for (const path of paths) { + if (path === '/') continue; + const exists = await this.client.exists(this.getRemotePath(path)); + if (!exists) return false; + } + + return true; + } + + async get(path: string): Promise { + if (path === '/') return; + await this.ensureConnected(); + const remotePath = this.getRemotePath(pathDirectory(path)); + const files = await this.client.list(remotePath); + const basename = pathBasename(path); + const entry = files.find(v => v.name === basename); + if (!entry) return; + return this.createStorageFile(path, entry); + } + + protected createStorageFile(path: string, fileInfo: FileInfo): StorageFile { + const file = new StorageFile(path, fileInfo.type !== 'd' ? FileType.File : FileType.Directory); + file.size = fileInfo.size; + file.lastModified = new Date(fileInfo.modifyTime); + file.visibility = this.mapModeToVisibility(file.type, fileInfo.rights); + + return file; + } + + async move(source: string, destination: string, reporter: Reporter): Promise { + await this.client.rename(this.getRemotePath(source), this.getRemotePath(destination)); + } + + async read(path: string, reporter: Reporter): Promise { + await this.ensureConnected(); + const remotePath = this.getRemotePath(path); + return await this.client.get(remotePath, undefined) as Buffer; + } + + async write(path: string, contents: Uint8Array, visibility: FileVisibility, reporter: Reporter): Promise { + await this.ensureConnected(); + await this.makeDirectory(pathDirectory(path), visibility); + await this.client.put(createReadable(contents), this.getRemotePath(path)); + await this.client.chmod(this.getRemotePath(path), this.getMode(FileType.File, visibility)); + } +} + +/** + * Best effort to parse date strings like `22 Oct 10 12:45` or `Oct 10 12:45` into a Date object. + */ +function parseCustomDateString(dateString: string): Date | undefined { + const currentYear = new Date().getFullYear(); + + const twoDigitYearMatch = dateString.match(/^\d{2}\s/); + const fourDigitYearMatch = dateString.match(/^\d{4}\s/); + + let fullDateString; + + if (twoDigitYearMatch) { + // Handle '22 Oct 10 12:45' format. + const twoDigitYear = twoDigitYearMatch[0].trim(); + const baseYear = currentYear.toString().substring(0, 2); // Get the first two digits of the current year. + fullDateString = `${baseYear}${twoDigitYear} ${dateString.substring(3)}`; + } else if (fourDigitYearMatch) { + // Handle '2022 Oct 10 12:45' format. + fullDateString = dateString; + } else { + // Handle 'Oct 10 12:45' format. + fullDateString = `${dateString} ${currentYear}`; + } + + return new Date(fullDateString); +} + +function createReadable(buffer: Uint8Array): Readable { + const stream = new Readable(); + stream.push(buffer); + stream.push(null); + return stream; +} diff --git a/packages/storage-sftp/tests/storage.spec.ts b/packages/storage-sftp/tests/storage.spec.ts new file mode 100644 index 000000000..47d0080c5 --- /dev/null +++ b/packages/storage-sftp/tests/storage.spec.ts @@ -0,0 +1,40 @@ +import { test } from '@jest/globals'; +import './storage.spec.js'; +import { setAdapterFactory } from '@deepkit/storage/test'; +import { StorageSftpAdapter } from '../src/sftp-adapter.js'; +import { platform } from 'os'; + +setAdapterFactory(async () => { + let adapter = new StorageSftpAdapter({ + host: 'localhost', + user: 'user', + password: '123', + root: 'upload' + });; + if (platform() === 'darwin') { + // docker run -d --name storage-sftp -p 22:22 -d atmoz/sftp user:123:::upload + adapter = new StorageSftpAdapter({ + host: 'storage-sftp.orb.local', + user: 'user', + password: '123', + root: 'upload' + }); + } + + //reset all files + await adapter.delete((await adapter.files('/')).map(v => v.path)); + + return adapter; +}); + +// since we import .storage.spec.js, all its tests are scheduled to run +// we define 'basic' here too, so we can easily run just this test. +// also necessary to have at least once test in this file, so that WebStorm +// detects the file as a test file. +test('url', () => undefined); +test('basic', () => undefined); +test('append/prepend', () => undefined); +test('visibility', () => undefined); +test('recursive', () => undefined); +test('copy', () => undefined); +test('move', () => undefined); diff --git a/packages/storage-sftp/tsconfig.esm.json b/packages/storage-sftp/tsconfig.esm.json new file mode 100644 index 000000000..e3a194731 --- /dev/null +++ b/packages/storage-sftp/tsconfig.esm.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/esm", + "module": "ES2020" + }, + "references": [ + { + "path": "../core/tsconfig.esm.json" + }, + { + "path": "../storage/tsconfig.esm.json" + } + ] +} diff --git a/packages/storage-sftp/tsconfig.json b/packages/storage-sftp/tsconfig.json new file mode 100644 index 000000000..cd3d04136 --- /dev/null +++ b/packages/storage-sftp/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "sourceMap": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "moduleResolution": "NodeNext", + "target": "es2018", + "module": "CommonJS", + "esModuleInterop": true, + "outDir": "./dist/cjs", + "declaration": true, + "composite": true + }, + "include": [ + "src", + "tests", + "index.ts" + ], + "references": [ + { + "path": "../core/tsconfig.json" + }, + { + "path": "../storage/tsconfig.json" + } + ] +} diff --git a/packages/storage/src/local-adapter.ts b/packages/storage/src/local-adapter.ts index b96cdb18d..fee27aab8 100644 --- a/packages/storage/src/local-adapter.ts +++ b/packages/storage/src/local-adapter.ts @@ -5,12 +5,12 @@ export interface StorageLocalAdapterOptions { root: string; permissions: { file: { - public: number; //default 0644 - private: number; //default 0.600 + public: number; //default 0o644 + private: number; //default 0o600 }, directory: { - public: number; //default 0755 - private: number; //default 0.700 + public: number; //default 0o755 + private: number; //default 0o700 } }; } @@ -51,9 +51,6 @@ export class StorageNodeLocalAdapter implements StorageAdapter { return this.fs; } - /** - * Mode is a number returned from Node's stat operation. - */ protected mapModeToVisibility(type: FileType, mode: number): FileVisibility { const permissions = this.options.permissions[type === FileType.File ? 'file' : 'directory']; const fileMode = mode & 0o777; @@ -61,7 +58,7 @@ export class StorageNodeLocalAdapter implements StorageAdapter { return 'private'; } - getMode(type: FileType, visibility: FileVisibility): number { + protected getMode(type: FileType, visibility: FileVisibility): number { const permissions = this.options.permissions[type === FileType.File ? 'file' : 'directory']; return visibility === 'public' ? permissions.public : permissions.private; } diff --git a/packages/storage/tests/storage.spec.ts b/packages/storage/tests/storage.spec.ts index 61c7ef475..e890e00a5 100644 --- a/packages/storage/tests/storage.spec.ts +++ b/packages/storage/tests/storage.spec.ts @@ -10,11 +10,17 @@ export function setAdapterFactory(factory: () => Promise) { test('url', async () => { const storage = new Storage(await adapterFactory(), { baseUrl: 'http://localhost/assets/' }); - if (storage.adapter.publicUrl) return; //has custom tests + if (storage.adapter.publicUrl) { + //has custom tests + await storage.close(); + return; + } //this test is about URL mapping feature from Storage const url = await storage.publicUrl('/file1.txt'); expect(url).toBe('http://localhost/assets/file1.txt'); + + await storage.close(); }); test('basic', async () => { @@ -113,6 +119,7 @@ test('visibility', async () => { expect(folder2).toMatchObject({ path: '/folder2', size: 0, visibility: 'private' }); } + await storage.setVisibility('file2.txt', 'public'); const file2b = await storage.get('/file2.txt'); expect(file2b).toMatchObject({ path: '/file2.txt', size: 9, lastModified: expect.any(Date), visibility: 'public' });