fix an unzip issue where file descriptor would not get closed properl…

…y, causing the request to hang forever
Holusion · Jan 9, 2025 · 5ef56d0 · 5ef56d0
1 parent bb08041
commit 5ef56d0
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 16 deletions.
diff --git a/source/server/routes/scenes/post.ts b/source/server/routes/scenes/post.ts
@@ -8,9 +8,10 @@ import yauzl, { Entry, ZipFile } from "yauzl";
 import { HTTPError } from "../../utils/errors.js";
 import { getMimeType } from "../../utils/filetypes.js";
 import { getVfs, getUser } from "../../utils/locals.js";
-import { Uid } from "../../utils/uid.js";
+import uid, { Uid } from "../../utils/uid.js";
 import { once } from "events";
 import { Readable } from "stream";
+import { finished, pipeline } from "stream/promises";
 
 
 interface ImportResults {
@@ -23,7 +24,7 @@ export default async function postScenes(req :Request, res :Response){
   let vfs = getVfs(req);
   let requester = getUser(req);
 
-  let file_name = Uid.toString(Uid.make());
+  let file_name = uid(12)+".zip";
   let tmpfile = path.join(vfs.uploadsDir, file_name);
   let results :ImportResults = {fail:[], ok:[]};
   let handle = await fs.open(tmpfile, "wx+");
@@ -88,11 +89,12 @@ export default async function postScenes(req :Request, res :Response){
       }else if(name.endsWith(".svx.json")){
         let data = Buffer.alloc(record.uncompressedSize), size = 0;
         let rs = await openZipEntry(record);
-        for await (let chunk of rs){
+        rs.on("data", (chunk)=>{
           chunk.copy(data, size);
           size += chunk.length;
-        }
-        await vfs.writeDoc(data.toString("utf8"), {scene, user_id: requester.uid, name, mime: "application/si-dpo-3d.document+json"});
+        });
+        await finished(rs);
+        await vfs.writeDoc(data, {scene, user_id: requester.uid, name, mime: "application/si-dpo-3d.document+json"});
       }else{
         //Add the file
         let rs = await openZipEntry(record);
@@ -103,7 +105,9 @@ export default async function postScenes(req :Request, res :Response){
     };
 
     zip.on("entry", (record)=>{
-      onEntry(record).then(()=> zip.readEntry(), (e)=>{
+      onEntry(record).then(()=>{
+        zip.readEntry()
+      }, (e)=>{
         console.log("unzip error :", e);
         results.fail.push(`Unzip error : ${e.message}`);
         zip.close();
@@ -113,5 +117,5 @@ export default async function postScenes(req :Request, res :Response){
     await once(zip, "close");
   }).finally(() => fs.rm(tmpfile, {force: true}));
 
-  res.status(200).send(results);
+  res.status(results.fail.length? 500:200).send(results);
 };
diff --git a/source/server/utils/uid.ts b/source/server/utils/uid.ts
@@ -32,7 +32,7 @@ export class Uid{
    * generate a random number that is a safe 48bit uint
    */
   static make() :number{
-    return randomBytes(6).readUIntLE(0,6);
+    return randomInt(0, 2**48-1);
   }
   /**
    * maps an unsigned integer ID to a base64url string

diff --git a/source/server/vfs/Files.ts b/source/server/vfs/Files.ts
@@ -2,13 +2,15 @@ import { constants, promises as fs } from "fs";
 import { createHash } from "crypto";
 import path from "path";
 import { NotFoundError, ConflictError, BadRequestError, InternalError } from "../utils/errors.js";
-import { Uid } from "../utils/uid.js";
+import uid, { Uid } from "../utils/uid.js";
 import BaseVfs from "./Base.js";
 import { DataStream, DocProps, FileProps, GetFileParams, GetFileResult, Stored, WriteDirParams, WriteDocParams, WriteFileParams } from "./types.js";
 
 import { Transaction } from "./helpers/db.js";
 import { FileHandle } from "fs/promises";
 import { Duplex, Readable, Transform } from "stream";
+import { pipeline } from "stream/promises";
+import { transform } from "typescript";
 
 interface DiskFileParams{
   size :number;
@@ -47,18 +49,24 @@ export default abstract class FilesVfs extends BaseVfs{
   ) :Promise<FileProps>{
 
     return this.createFile(params, async ({id})=>{
-      let file_name = Uid.toString(id)
+      let file_name = Uid.toString(id)+"_"+uid(6);
       let tmpfile = path.join(this.uploadsDir, file_name);
       let handle = await fs.open(tmpfile, constants.O_RDWR|constants.O_CREAT|constants.O_EXCL);
       let hashsum = createHash("sha256");
       let size = 0;
       try{
-        for await (let d of dataStream){
-          let {bytesWritten} = await handle.write(d);
-          size += bytesWritten;
-          hashsum.update(d);
-        }
-        //hash.digest("base64url")
+        let ws = handle.createWriteStream();
+        await pipeline(
+          dataStream,
+          new Transform({
+            transform(chunk, encoding, callback){
+              hashsum.update(chunk);
+              size += chunk.length;
+              callback(null, chunk);
+            }
+          }),
+          ws,
+        );
         let hash = hashsum.digest("base64url");
         let destfile = path.join(this.objectsDir, hash);
         try{