From b36495adac57c1907ff807548737816ce1ef1d53 Mon Sep 17 00:00:00 2001 From: kukhariev Date: Wed, 1 Sep 2021 00:51:30 +0300 Subject: [PATCH] feat: extended completed file --- examples/express.ts | 66 ++++++---------------- packages/core/src/storages/disk-storage.ts | 34 ++++++++++- packages/core/src/utils/fs.ts | 38 ++++++++++++- packages/s3/src/s3-storage.ts | 12 ++-- test/disk-storage.spec.ts | 4 +- test/multipart.spec.ts | 4 +- test/s3-storage.spec.ts | 2 +- test/tus.spec.ts | 10 +++- test/uploadx.spec.ts | 4 +- 9 files changed, 107 insertions(+), 67 deletions(-) diff --git a/examples/express.ts b/examples/express.ts index baa56a5c..f744bb65 100644 --- a/examples/express.ts +++ b/examples/express.ts @@ -1,57 +1,27 @@ import * as express from 'express'; -import { DiskFile, DiskStorage, OnComplete, uploadx, UploadxResponse } from 'node-uploadx'; +import { DiskFile, uploadx } from 'node-uploadx'; +import { join } from 'path'; const app = express(); -const auth: express.Handler = (req, res, next) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - (req as any)['user'] = { id: '92be348f-172d-5f69-840d-100f79e4d1ef' }; - next(); +const onComplete: express.RequestHandler = async (req, res, next) => { + const file = req.body as DiskFile; + await file.lock(() => res.status(423).json({ message: 'processing' })); + const sha1 = await file.hash('sha1'); + await file.move(join('upload', file.originalName)); + await file.lock(() => res.json({ ...file, sha1 })); + await file.delete(); + return res.json({ ...file, sha1 }); }; -app.use(auth); - -type OnCompleteBody = { - message: string; - id: string; -}; - -const onComplete: OnComplete> = file => { - const message = `File upload is finished, path: ${file.name}`; - console.log(message); - return { - statusCode: 200, - message, - id: file.id, - headers: { ETag: file.id } - }; -}; - -const storage = new DiskStorage({ - directory: 'upload', - maxMetadataSize: '1mb', - onComplete, - expiration: { maxAge: '1h', purgeInterval: '10min' }, - validation: { - mime: { value: ['video/*'], response: [415, { message: 'video only' }] }, - size: { - value: 500_000_000, - isValid(file) { - this.response = [ - 412, - { message: `The file size(${file.size}) is larger than ${this.value as number} bytes` } - ]; - return file.size <= this.value; - } - }, - mtime: { - isValid: file => !!file.metadata.lastModified, - response: [403, { message: 'Missing `lastModified` property' }] - } - } -}); - -app.use('/files', uploadx({ storage })); +app.all( + '/files', + uploadx.upload({ + directory: 'upload', + expiration: { maxAge: '12h', purgeInterval: '1h' } + }), + onComplete +); app.listen(3002, () => { console.log('listening on port:', 3002); diff --git a/packages/core/src/storages/disk-storage.ts b/packages/core/src/storages/disk-storage.ts index c6a5d58f..f6671d8c 100644 --- a/packages/core/src/storages/disk-storage.ts +++ b/packages/core/src/storages/disk-storage.ts @@ -1,6 +1,15 @@ import * as http from 'http'; import { resolve as pathResolve } from 'path'; -import { ensureFile, ERRORS, fail, fsp, getWriteStream, HttpError } from '../utils'; +import { + ensureFile, + ERRORS, + fail, + fileChecksum, + fsp, + getWriteStream, + HttpError, + move +} from '../utils'; import { File, FileInit, FilePart, hasContent, isCompleted, isValidPart } from './file'; import { BaseStorage, BaseStorageOptions } from './storage'; import { METAFILE_EXTNAME, MetaStorage } from './meta-storage'; @@ -8,7 +17,13 @@ import { LocalMetaStorage, LocalMetaStorageOptions } from './local-meta-storage' const INVALID_OFFSET = -1; -export class DiskFile extends File {} +export interface DiskFile extends File { + lock: (lockFn: () => any) => Promise; + move: (dest: string) => Promise; + copy: (dest: string) => Promise; + delete: () => Promise; + hash: (algorithm?: 'sha1' | 'md5', encoding?: 'hex' | 'base64') => Promise; +} export type DiskStorageOptions = BaseStorageOptions & { /** @@ -51,8 +66,20 @@ export class DiskStorage extends BaseStorage { return super.normalizeError(error); } + buildCompletedFile(file: DiskFile): DiskFile { + const completed = { ...file }; + completed.lock = async lockFn => Promise.resolve('TODO:'); + completed.delete = () => this.delete(file.name); + completed.hash = (algorithm?: 'sha1' | 'md5', encoding?: 'hex' | 'base64') => + fileChecksum(this.getFilePath(file.name), algorithm, encoding); + completed.copy = async (dest: string) => fsp.copyFile(this.getFilePath(file.name), dest); + completed.move = async (dest: string) => move(this.getFilePath(file.name), dest); + + return completed; + } + async create(req: http.IncomingMessage, fileInit: FileInit): Promise { - const file = new DiskFile(fileInit); + const file = new File(fileInit) as DiskFile; file.name = this.namingFunction(file); await this.validate(file); const path = this.getFilePath(file.name); @@ -72,6 +99,7 @@ export class DiskStorage extends BaseStorage { if (file.bytesWritten === INVALID_OFFSET) return fail(ERRORS.FILE_CONFLICT); if (isCompleted(file)) { await this.saveMeta(file); + return this.buildCompletedFile(file); } return file; } catch (err) { diff --git a/packages/core/src/utils/fs.ts b/packages/core/src/utils/fs.ts index 1d97a1e0..0755f5a9 100644 --- a/packages/core/src/utils/fs.ts +++ b/packages/core/src/utils/fs.ts @@ -1,12 +1,44 @@ -import { createWriteStream, promises as fsp, WriteStream } from 'fs'; +import { createHash, HexBase64Latin1Encoding } from 'crypto'; +import { createReadStream, createWriteStream, promises as fsp, WriteStream } from 'fs'; import { dirname, posix } from 'path'; +function isError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error; +} + +export function fileChecksum( + filePath: string, + algorithm: 'sha1' | 'md5' = 'md5', + encoding: HexBase64Latin1Encoding = 'hex' +): Promise { + return new Promise(resolve => { + const hash = createHash(algorithm); + createReadStream(filePath) + .on('data', data => hash.update(data)) + .on('end', () => resolve(hash.digest(encoding))); + }); +} + +export async function copy(src: string, dest: string): Promise { + return fsp.copyFile(src, dest); +} + +export async function move(src: string, dest: string): Promise { + try { + await fsp.rename(src, dest); + } catch (e) { + if (isError(e) && e.code === 'EXDEV') { + await copy(src, dest); + await fsp.unlink(src); + } + } +} /** * Ensures that the directory exists * @param dir */ -export async function ensureDir(dir: string): Promise { - await fsp.mkdir(dir, { recursive: true }); +export function ensureDir(dir: string): Promise { + return fsp.mkdir(dir, { recursive: true }); } /** diff --git a/packages/s3/src/s3-storage.ts b/packages/s3/src/s3-storage.ts index 89f19938..86dfb5e4 100644 --- a/packages/s3/src/s3-storage.ts +++ b/packages/s3/src/s3-storage.ts @@ -37,10 +37,14 @@ import { S3MetaStorage, S3MetaStorageOptions } from './s3-meta-storage'; const BUCKET_NAME = 'node-uploadx'; -export class S3File extends File { +export interface S3File extends File { Parts?: Part[]; - UploadId = ''; + UploadId?: string; uri?: string; + lock: (lockFn: () => any) => Promise; + move: (dest: any) => Promise; + copy: (dest: any) => Promise; + delete: () => Promise; } export type S3StorageOptions = BaseStorageOptions & @@ -121,7 +125,7 @@ export class S3Storage extends BaseStorage { } async create(req: http.IncomingMessage, config: FileInit): Promise { - const file = new S3File(config); + const file = new File(config) as S3File; file.name = this.namingFunction(file); await this.validate(file); try { @@ -226,7 +230,7 @@ export class S3Storage extends BaseStorage { } private _checkBucket(): void { - this.client.send(new HeadBucketCommand({ Bucket: this.bucket }), (err: AWSError, data) => { + this.client.send(new HeadBucketCommand({ Bucket: this.bucket }), (err: AWSError) => { if (err) { throw err; } diff --git a/test/disk-storage.spec.ts b/test/disk-storage.spec.ts index 068b8566..cdc6f839 100644 --- a/test/disk-storage.spec.ts +++ b/test/disk-storage.spec.ts @@ -1,5 +1,5 @@ import { posix } from 'path'; -import { DiskStorage, fsp } from '../packages/core/src'; +import { DiskStorage, DiskStorageOptions, fsp } from '../packages/core/src'; import { storageOptions } from './fixtures'; import { FileWriteStream, RequestReadStream } from './fixtures/streams'; import { filename, metafile, testfile } from './fixtures/testfile'; @@ -26,7 +26,7 @@ jest.mock('../packages/core/src/utils/fs', () => { }); describe('DiskStorage', () => { - const options = { ...storageOptions, directory }; + const options = { ...storageOptions, directory } as DiskStorageOptions; let storage: DiskStorage; let mockReadable: RequestReadStream; const createFile = (): Promise => { diff --git a/test/multipart.spec.ts b/test/multipart.spec.ts index 614417e2..564aa1f8 100644 --- a/test/multipart.spec.ts +++ b/test/multipart.spec.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import { join } from 'path'; import * as request from 'supertest'; -import { multipart } from '../packages/core/src'; +import { multipart, DiskStorageOptions } from '../packages/core/src'; import { root, storageOptions } from './fixtures'; import { app } from './fixtures/app'; import { metadata, srcpath } from './fixtures/testfile'; @@ -12,7 +12,7 @@ describe('::Multipart', () => { const files: string[] = []; const basePath = '/multipart'; const directory = join(root, 'multipart'); - const opts = { ...storageOptions, directory }; + const opts = { ...storageOptions, directory } as DiskStorageOptions; app.use(basePath, multipart(opts)); beforeAll(() => cleanup(directory)); diff --git a/test/s3-storage.spec.ts b/test/s3-storage.spec.ts index f194c278..2be58ba4 100644 --- a/test/s3-storage.spec.ts +++ b/test/s3-storage.spec.ts @@ -36,7 +36,7 @@ describe('S3Storage', () => { ...testfile, UploadId: '123456789', Parts: [] - }; + } as unknown as S3File; }); describe('.create()', () => { diff --git a/test/tus.spec.ts b/test/tus.spec.ts index d2183d06..61b43d86 100644 --- a/test/tus.spec.ts +++ b/test/tus.spec.ts @@ -2,7 +2,13 @@ import * as fs from 'fs'; import { join } from 'path'; import * as request from 'supertest'; -import { BaseStorage, serializeMetadata, tus, TUS_RESUMABLE } from '@uploadx/core'; +import { + BaseStorage, + DiskStorageOptions, + serializeMetadata, + tus, + TUS_RESUMABLE +} from '@uploadx/core'; import { root, storageOptions } from './fixtures'; import { app } from './fixtures/app'; import { metadata, srcpath } from './fixtures/testfile'; @@ -12,7 +18,7 @@ describe('::Tus', () => { let uri: string; const basePath = '/tus'; const directory = join(root, 'tus'); - const opts = { ...storageOptions, directory }; + const opts = { ...storageOptions, directory } as DiskStorageOptions; app.use(basePath, tus(opts)); beforeAll(() => cleanup(directory)); diff --git a/test/uploadx.spec.ts b/test/uploadx.spec.ts index 98f230e5..e51a920e 100644 --- a/test/uploadx.spec.ts +++ b/test/uploadx.spec.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import { join } from 'path'; import * as request from 'supertest'; -import { uploadx } from '../packages/core/src'; +import { uploadx, DiskStorageOptions } from '../packages/core/src'; import { root, storageOptions, userPrefix } from './fixtures'; import { app } from './fixtures/app'; import { metadata, srcpath } from './fixtures/testfile'; @@ -13,7 +13,7 @@ describe('::Uploadx', () => { let start: number; const basePath = '/uploadx'; const directory = join(root, 'uploadx'); - const opts = { ...storageOptions, directory, maxMetadataSize: 250 }; + const opts = { ...storageOptions, directory, maxMetadataSize: 250 } as DiskStorageOptions; app.use(basePath, uploadx(opts)); beforeAll(() => cleanup(directory));