From 895ccae06cb5c39bb62342f48b2c6cd9ed502c8a Mon Sep 17 00:00:00 2001 From: "Marc J. Schmidt" Date: Sun, 8 Oct 2023 16:44:38 +0200 Subject: [PATCH] feature(storage): new storage abstraction more adapters to come --- package.json | 1 + packages/storage/.npmignore | 0 packages/storage/README.md | 14 ++ packages/storage/dist/.gitkeep | 0 packages/storage/index.ts | 1 + packages/storage/package.json | 50 +++++ packages/storage/src/local-adapter.ts | 104 +++++++++ packages/storage/src/storage.ts | 289 +++++++++++++++++++++++++ packages/storage/tests/local.spec.ts | 19 ++ packages/storage/tests/storage.spec.ts | 100 +++++++++ packages/storage/tsconfig.esm.json | 12 + packages/storage/tsconfig.json | 28 +++ tsconfig.esm.json | 3 + tsconfig.json | 3 + 14 files changed, 624 insertions(+) create mode 100644 packages/storage/.npmignore create mode 100644 packages/storage/README.md create mode 100644 packages/storage/dist/.gitkeep create mode 100644 packages/storage/index.ts create mode 100644 packages/storage/package.json create mode 100644 packages/storage/src/local-adapter.ts create mode 100644 packages/storage/src/storage.ts create mode 100644 packages/storage/tests/local.spec.ts create mode 100644 packages/storage/tests/storage.spec.ts create mode 100644 packages/storage/tsconfig.esm.json create mode 100644 packages/storage/tsconfig.json diff --git a/package.json b/package.json index be0ec2b5a..41b18e630 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "packages/template", "packages/injector", "packages/mongo", + "packages/storage", "packages/sql", "packages/mysql", "packages/postgres", diff --git a/packages/storage/.npmignore b/packages/storage/.npmignore new file mode 100644 index 000000000..e69de29bb diff --git a/packages/storage/README.md b/packages/storage/README.md new file mode 100644 index 000000000..dbd412bc1 --- /dev/null +++ b/packages/storage/README.md @@ -0,0 +1,14 @@ +# skeleton package + +This package can be copied when a new package should be created. + +### Steps after copying: + +- Adjust "name", "description", and "private" in `package.json`. +- Adjust README.md +- Put this package into root `/package.json` "jest.references". +- Put this package into root `/tsconfig.json` "references". +- Put this package into root `/tsconfig.esm.json` "references". +- Add dependencies to `package.json` and run `node sync-tsconfig-deps.js` to adjust tsconfig automatically. +- Add to .github/workflows/main.yml tsc build step if necessary. +- Add to typedoc build in deepkit-website if necessary. diff --git a/packages/storage/dist/.gitkeep b/packages/storage/dist/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/packages/storage/index.ts b/packages/storage/index.ts new file mode 100644 index 000000000..3a7aa7071 --- /dev/null +++ b/packages/storage/index.ts @@ -0,0 +1 @@ +export * from './src/storage.js'; diff --git a/packages/storage/package.json b/packages/storage/package.json new file mode 100644 index 000000000..78f31bbf9 --- /dev/null +++ b/packages/storage/package.json @@ -0,0 +1,50 @@ +{ + "name": "@deepkit/storage", + "version": "1.0.1-alpha.13", + "description": "Fileystem abstraction Deepkit", + "type": "commonjs", + "main": "./dist/cjs/index.js", + "module": "./dist/esm/index.js", + "types": "./dist/cjs/index.d.ts", + "exports": { + ".": { + "types": "./dist/cjs/index.d.ts", + "require": "./dist/cjs/index.js", + "default": "./dist/esm/index.js" + } + }, + "repository": "https://github.com/deepkit/deepkit-framework", + "author": "Marc J. Schmidt ", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json" + }, + "peerDependencies": { + "@deepkit/core": "^1.0.1-alpha.13" + }, + "dependencies": { + }, + "devDependencies": { + "@deepkit/core": "^1.0.1-alpha.13" + }, + "jest": { + "testEnvironment": "node", + "transform": { + "^.+\\.(ts|tsx)$": [ + "ts-jest", + { + "tsconfig": "/tsconfig.json" + } + ] + }, + "moduleNameMapper": { + "(.+)\\.js": "$1" + }, + "testMatch": [ + "**/tests/**/*.spec.ts" + ] + } +} diff --git a/packages/storage/src/local-adapter.ts b/packages/storage/src/local-adapter.ts new file mode 100644 index 000000000..976b85e2a --- /dev/null +++ b/packages/storage/src/local-adapter.ts @@ -0,0 +1,104 @@ +import { File, FileType, pathDirectory, pathNormalize, Reporter, StorageAdapter } from './storage.js'; +import type * as fs from 'fs/promises'; + +export class StorageNodeLocalAdapter implements StorageAdapter { + fs?: typeof fs; + + constructor(public path: string) { + this.path = pathNormalize(path); + } + + protected async getFs(): Promise { + if (!this.fs) this.fs = await import('fs/promises'); + return this.fs; + } + + getPath(path: string): string { + return this.path + path; + } + + async copy(source: string, destination: string, reporter: Reporter): Promise { + source = this.getPath(source); + destination = this.getPath(destination); + const fs = await this.getFs(); + await fs.cp(source, destination, { recursive: true }); + } + + async delete(path: string): Promise { + path = this.getPath(path); + const fs = await this.getFs(); + await fs.rm(path); + } + + async deleteDirectory(path: string, reporter: Reporter): Promise { + path = this.getPath(path); + const fs = await this.getFs(); + await fs.rm(path, { recursive: true }); + } + + async exists(path: string): Promise { + path = this.getPath(path); + const fs = await this.getFs(); + try { + const res = await fs.stat(path); + return res.isFile() || res.isDirectory(); + } catch (error: any) { + return false; + } + } + + async files(path: string): Promise { + const localPath = this.getPath(path); + const files: File[] = []; + const fs = await this.getFs(); + + for (const name of await fs.readdir(localPath)) { + const file = new File(path + '/' + name); + const stat = await fs.stat(localPath + '/' + name); + file.size = stat.size; + file.lastModified = new Date(stat.mtime); + file.type = stat.isFile() ? FileType.File : FileType.Directory; + files.push(file); + } + + return files; + } + + async get(path: string): Promise { + const localPath = this.getPath(path); + const fs = await this.getFs(); + const file = new File(path); + try { + const stat = await fs.stat(localPath); + file.size = stat.size; + file.lastModified = new Date(stat.mtime); + file.type = stat.isFile() ? FileType.File : FileType.Directory; + return file; + } catch (error: any) { + return undefined; + } + } + + async move(source: string, destination: string, reporter: Reporter): Promise { + source = this.getPath(source); + destination = this.getPath(destination); + const fs = await this.getFs(); + await fs.rename(source, destination); + } + + async read(path: string, reporter: Reporter): Promise { + path = this.getPath(path); + const fs = await this.getFs(); + const content = await fs.readFile(path); + return content; + } + + async write(path: string, contents: Uint8Array, reporter: Reporter): Promise { + path = this.getPath(path); + const fs = await this.getFs(); + await fs.mkdir(pathDirectory(path), { recursive: true }); + await fs.writeFile(path, contents); + } +} + +export const StorageLocalAdapter = StorageNodeLocalAdapter; diff --git a/packages/storage/src/storage.ts b/packages/storage/src/storage.ts new file mode 100644 index 000000000..590d7defa --- /dev/null +++ b/packages/storage/src/storage.ts @@ -0,0 +1,289 @@ +import { asyncOperation } from '@deepkit/core'; + +export enum FileType { + File, + Directory, + SymbolicLink, + Unknown, +} + +export function pathNormalize(path: string): string { + path = path[0] !== '/' ? '/' + path : path; + path = path.length > 1 && path[path.length - 1] === '/' ? path.slice(0, -1) : path; + return path.replace(/\/+/g, '/'); +} + +export function pathDirectory(path: string): string { + if (path === '/') return '/'; + const lastSlash = path.lastIndexOf('/'); + return lastSlash === -1 ? '' : path.slice(0, lastSlash); +} + +export class File { + public size?: number; + public type: FileType = FileType.File; + public lastModified?: Date; + + constructor(public path: string) { + this.path = pathNormalize(path); + } + + isFile(): boolean { + return this.type === FileType.File; + } + + isDirectory(): boolean { + return this.type === FileType.Directory; + } + + get name() { + const lastSlash = this.path.lastIndexOf('/'); + return lastSlash === -1 ? this.path : this.path.slice(lastSlash + 1); + } + + get directory() { + const lastSlash = this.path.lastIndexOf('/'); + return lastSlash === -1 ? '' : this.path.slice(0, lastSlash); + } + + get extension() { + const lastDot = this.path.lastIndexOf('.'); + return lastDot === -1 ? '' : this.path.slice(lastDot + 1); + } +} + +export interface Progress extends Promise { + onProgress(callback: (loaded: number, total: number) => void): this; + + abort(): Promise; +} + +export interface Storage { + // write(path: string, contents: string | Uint8Array): Progress; + // + // read(path: string): Progress; + // + // delete(path: string): Promise; + // + // deleteDirectory(path: string): Promise; + + // exists(path: string): Promise; + + // files(path: string): Promise; + + allFiles(path: string): Promise; + + directories(path: string): Promise; + + allDirectories(path: string): Promise; + + copy(source: string, destination: string): Promise; + + move(source: string, destination: string): Promise; + + sizeDirectory(path: string): Promise; +} + +export interface StorageAdapter { + files(path: string): Promise; + + write(path: string, contents: Uint8Array, reporter: Reporter): Promise; + + read(path: string, reporter: Reporter): Promise; + + get(path: string): Promise; + + exists(path: string): Promise; + + delete(path: string): Promise; + + deleteDirectory(path: string, reporter: Reporter): Promise; + + copy(source: string, destination: string, reporter: Reporter): Promise; + + move(source: string, destination: string, reporter: Reporter): Promise; +} + +export class FileNotFound extends Error { +} + +export type Reporter = { progress: (loaded: number, total: number) => void, onAbort: () => Promise }; + +export function createProgress(callback: (reporter: Reporter) => Promise): Progress { + const callbacks: ((loaded: number, total: number) => void)[] = []; + + const reporter = { + progress: (loaded: number, total: number) => { + for (const callback of callbacks) callback(loaded, total); + }, + onAbort: () => Promise.resolve() + }; + + const promise = asyncOperation(async (resolve, reject) => { + resolve(await callback(reporter)); + }) as Progress; + + promise.onProgress = (callback: (loaded: number, total: number) => void) => { + callbacks.push(callback); + return promise; + }; + + promise.abort = async () => { + await reporter.onAbort(); + }; + + return promise; +} + +export class StorageMemoryAdapter implements StorageAdapter { + protected memory: { file: File, contents: Uint8Array }[] = []; + + async files(path: string): Promise { + return this.memory.filter(file => file.file.path.startsWith(path)).map(v => v.file); + } + + async write(path: string, contents: Uint8Array, reporter: Reporter): Promise { + let file = this.memory.find(file => file.file.path === path); + if (!file) { + file = { file: new File(path), contents }; + this.memory.push(file); + } + file.contents = contents; + file.file.size = contents.length; + file.file.lastModified = new Date(); + } + + async read(path: string, reporter: Reporter): Promise { + const file = this.memory.find(file => file.file.path === path); + if (!file) throw new FileNotFound('File not found'); + return file.contents; + } + + async exists(path: string): Promise { + return !!this.memory.find(file => file.file.path === path); + } + + async delete(path: string): Promise { + const index = this.memory.findIndex(file => file.file.path === path); + if (index === -1) throw new FileNotFound('File not found'); + this.memory.splice(index, 1); + } + + async deleteDirectory(path: string, reporter: Reporter): Promise { + const files = this.memory.filter(file => file.file.path.startsWith(path)); + reporter.progress(0, files.length); + let i = 0; + for (const file of files) { + this.memory.splice(this.memory.indexOf(file), 1); + reporter.progress(++i, files.length); + } + } + + async get(path: string): Promise { + return this.memory.find(file => file.file.path === path)?.file; + } + + async copy(source: string, destination: string, reporter: Reporter): Promise { + const files = this.memory.filter(file => file.file.path.startsWith(source)); + reporter.progress(0, files.length); + let i = 0; + for (const file of files) { + const newPath = destination + file.file.path.slice(source.length); + this.memory.push({ file: new File(newPath), contents: file.contents }); + reporter.progress(++i, files.length); + } + } + + async move(source: string, destination: string, reporter: Reporter): Promise { + const files = this.memory.filter(file => file.file.path.startsWith(source)); + reporter.progress(0, files.length); + let i = 0; + for (const file of files) { + const newPath = destination + file.file.path.slice(source.length); + file.file.path = newPath; + reporter.progress(++i, files.length); + } + } +} + +export class Storage { + constructor(public adapter: StorageAdapter) { + } + + protected normalizePath(path: string): string { + return pathNormalize(path); + } + + files(path: string): Promise { + path = this.normalizePath(path); + return this.adapter.files(path); + } + + write(path: string, content: Uint8Array | string): Progress { + path = this.normalizePath(path); + const buffer = typeof content === 'string' ? new TextEncoder().encode(content) : content; + return createProgress(async (reporter) => { + return await this.adapter.write(path, buffer, reporter); + }); + } + + read(path: string): Progress { + path = this.normalizePath(path); + return createProgress(async (reporter) => { + return await this.adapter.read(path, reporter); + }); + } + + readAsText(path: string): Progress { + path = this.normalizePath(path); + return createProgress(async (reporter) => { + const contents = await this.adapter.read(path, reporter); + return new TextDecoder().decode(contents); + }); + } + + async get(path: string): Promise { + path = this.normalizePath(path); + const file = await this.adapter.get(path); + if (!file) throw new FileNotFound('File not found'); + return file; + } + + getOrUndefined(path: string): Promise { + path = this.normalizePath(path); + return this.adapter.get(path); + } + + exists(path: string): Promise { + path = this.normalizePath(path); + return this.adapter.exists(path); + } + + delete(path: string): Promise { + path = this.normalizePath(path); + return this.adapter.delete(path); + } + + deleteDirectory(path: string): Progress { + path = this.normalizePath(path); + return createProgress(async (reporter) => { + return this.adapter.deleteDirectory(path, reporter); + }); + } + + copy(source: string, destination: string): Progress { + source = this.normalizePath(source); + destination = this.normalizePath(destination); + return createProgress(async (reporter) => { + return this.adapter.copy(source, destination, reporter); + }); + } + + move(source: string, destination: string): Progress { + source = this.normalizePath(source); + destination = this.normalizePath(destination); + return createProgress(async (reporter) => { + return this.adapter.move(source, destination, reporter); + }); + } +} diff --git a/packages/storage/tests/local.spec.ts b/packages/storage/tests/local.spec.ts new file mode 100644 index 000000000..c896773e8 --- /dev/null +++ b/packages/storage/tests/local.spec.ts @@ -0,0 +1,19 @@ +import { test } from '@jest/globals'; +import './storage.spec.js'; +import { setAdapterFactory } from './storage.spec.js'; +import { StorageLocalAdapter } from '../src/local-adapter.js'; +import { mkdtempSync } from 'fs'; +import { tmpdir } from 'os'; + +setAdapterFactory(async () => { + const tmp = mkdtempSync(tmpdir() + '/storage-test-'); + return new StorageLocalAdapter(tmp); +}); + +// since we import .storage.spec.js, all its tests are scheduled to run +// we define 'basic' here too, so we can easily run just this test. +// also necessary to have at least once test in this file, so that WebStorm +// detects the file as a test file. +test('basic', () => undefined); +test('copy', () => undefined); +test('move', () => undefined); diff --git a/packages/storage/tests/storage.spec.ts b/packages/storage/tests/storage.spec.ts new file mode 100644 index 000000000..72bc8d157 --- /dev/null +++ b/packages/storage/tests/storage.spec.ts @@ -0,0 +1,100 @@ +import { expect, test } from '@jest/globals'; +import { Storage, StorageAdapter, StorageMemoryAdapter } from '../src/storage.js'; + +export let adapterFactory: () => Promise = async () => new StorageMemoryAdapter; + +export function setAdapterFactory(factory: () => Promise) { + adapterFactory = factory; +} + +test('basic', async () => { + const storage = new Storage(await adapterFactory()); + + const files = await storage.files('/'); + expect(files).toEqual([]); + + await storage.write('/file1.txt', 'contents1'); + await storage.write('/file2.txt', 'contents2'); + await storage.write('/file3.txt', 'abc'); + + const files2 = await storage.files('/'); + expect(files2).toMatchObject([ + { path: '/file1.txt', size: 9, lastModified: expect.any(Date) }, + { path: '/file2.txt', size: 9, lastModified: expect.any(Date) }, + { path: '/file3.txt', size: 3, lastModified: expect.any(Date) }, + ]); + + const content = await storage.readAsText('/file1.txt'); + expect(content).toBe('contents1'); + + const file = await storage.get('/file1.txt'); + expect(file).toMatchObject({ path: '/file1.txt', size: 9, lastModified: expect.any(Date) }); + + await expect(() => storage.get('/file4.txt')).rejects.toThrowError('File not found'); + expect(await storage.getOrUndefined('/file4.txt')).toBe(undefined); + + expect(await storage.exists('/file1.txt')).toBe(true); + expect(await storage.exists('/file2.txt')).toBe(true); + expect(await storage.exists('/file3.txt')).toBe(true); + expect(await storage.exists('//file3.txt')).toBe(true); + expect(await storage.exists('/file3.txt/')).toBe(true); + expect(await storage.exists('//file3.txt/')).toBe(true); + + expect(await storage.exists('/file4.txt')).toBe(false); + expect(await storage.exists('//file4.txt')).toBe(false); + + await storage.delete('/file1.txt'); + await storage.delete('/file2.txt'); + + expect(await storage.exists('/file1.txt')).toBe(false); + expect(await storage.exists('/file2.txt')).toBe(false); + expect(await storage.exists('/file3.txt')).toBe(true); + + await storage.deleteDirectory('/'); + expect(await storage.exists('/file3.txt')).toBe(false); +}); + +test('copy', async () => { + const storage = new Storage(await adapterFactory()); + + await storage.write('/file1.txt', 'contents1'); + await storage.write('/folder/file1.txt', 'contents2'); + await storage.write('/folder/file2.txt', 'contents3'); + + await storage.copy('/file1.txt', '/file2.txt'); + await storage.copy('/folder/file1.txt', '/folder/file3.txt'); + + expect(await storage.exists('/file1.txt')).toBe(true); + expect(await storage.exists('/file2.txt')).toBe(true); + + expect(await storage.readAsText('/file1.txt')).toBe('contents1'); + expect(await storage.readAsText('/file2.txt')).toBe('contents1'); + + await storage.copy('/folder', '/folder2'); + expect(await storage.exists('/folder/file1.txt')).toBe(true); + expect(await storage.exists('/folder2/file1.txt')).toBe(true); + expect(await storage.exists('/folder2/file2.txt')).toBe(true); + expect(await storage.exists('/folder2/file3.txt')).toBe(true); +}); + +test('move', async () => { + const storage = new Storage(await adapterFactory()); + + await storage.write('/file1.txt', 'contents1'); + await storage.write('/folder/file1.txt', 'contents2'); + await storage.write('/folder/file2.txt', 'contents3'); + + await storage.move('/file1.txt', '/file2.txt'); + await storage.move('/folder/file1.txt', '/folder/file3.txt'); + + expect(await storage.exists('/file1.txt')).toBe(false); + expect(await storage.exists('/file2.txt')).toBe(true); + + expect(await storage.readAsText('/file2.txt')).toBe('contents1'); + + await storage.move('/folder', '/folder2'); + expect(await storage.exists('/folder/file1.txt')).toBe(false); + expect(await storage.exists('/folder/file3.txt')).toBe(false); + expect(await storage.exists('/folder2/file2.txt')).toBe(true); + expect(await storage.exists('/folder2/file3.txt')).toBe(true); +}); diff --git a/packages/storage/tsconfig.esm.json b/packages/storage/tsconfig.esm.json new file mode 100644 index 000000000..d91532557 --- /dev/null +++ b/packages/storage/tsconfig.esm.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist/esm", + "module": "ES2020" + }, + "references": [ + { + "path": "../core/tsconfig.esm.json" + } + ] +} \ No newline at end of file diff --git a/packages/storage/tsconfig.json b/packages/storage/tsconfig.json new file mode 100644 index 000000000..a98ec6b42 --- /dev/null +++ b/packages/storage/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "sourceMap": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "moduleResolution": "node", + "target": "es2018", + "module": "CommonJS", + "esModuleInterop": true, + "outDir": "./dist/cjs", + "declaration": true, + "composite": true + }, + "include": [ + "src", + "tests", + "index.ts" + ], + "references": [ + { + "path": "../core/tsconfig.json" + } + ] +} \ No newline at end of file diff --git a/tsconfig.esm.json b/tsconfig.esm.json index 793c6bb61..7ff1db734 100644 --- a/tsconfig.esm.json +++ b/tsconfig.esm.json @@ -93,6 +93,9 @@ }, { "path": "packages/bun/tsconfig.esm.json" + }, + { + "path": "packages/storage/tsconfig.esm.json" } ] } diff --git a/tsconfig.json b/tsconfig.json index 093f2f079..daa9f077e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -111,6 +111,9 @@ }, { "path": "packages/bun/tsconfig.json" + }, + { + "path": "packages/storage/tsconfig.json" } ] }