add list and upload endpoints

hzrd149 · Feb 17, 2024 · 266d349 · 266d349
1 parent 0946563
commit 266d349
Show file tree

Hide file tree

Showing 14 changed files with 364 additions and 51 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -2,3 +2,4 @@ data
 .env
 node_modules
 .github
+database.json
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@ node_modules
 data
 .env
 build
+database.json
diff --git a/README.md b/README.md
@@ -1,3 +1,8 @@
 # blobstr
 
-A http file server with a nostr twist
+Generic blob storage and retrieval for nostr
+
+## TODO
+
+- verify auth events
+- write up simple API spec
diff --git a/config.yml b/config.yml
@@ -1,3 +1,5 @@
+publicDomain: http://localhost:3000
+
 discovery:
   # find files by querying nostr relays
   nostr:
@@ -13,8 +15,8 @@ discovery:
     enabled: true
     domains:
       - https://cdn.satellite.earth
-      - https://image.nostr.build
-      - https://video.nostr.build
+      # - https://image.nostr.build
+      # - https://video.nostr.build
 
 cache:
   dir: ./data
@@ -27,3 +29,13 @@ cache:
       expiration: 5 days
     - type: "*"
       expiration: 1 day
+
+upload:
+  enabled: true
+  rules:
+    - type: "*"
+      expiration: 1d
+
+tor:
+  enabled: false
+  proxy: ""
diff --git a/package.json b/package.json
@@ -1,14 +1,15 @@
 {
   "name": "blobstr",
   "version": "0.1.0",
+  "description": "Generic blob storage and retrieval for nostr",
   "main": "index.js",
   "type": "module",
   "author": "hzrd149",
   "license": "MIT",
   "scripts": {
     "start": "node src/index.js",
     "build": "tsc",
-    "dev": "DEBUG=* nodemon --ignore '**/data/**' build/index.js",
+    "dev": "DEBUG=* nodemon -i '**/data/**' -i '**/database.json' -i '**/src/**' build/index.js",
     "format": "prettier -w ."
   },
   "bin": "build/index.js",
@@ -26,7 +27,9 @@
     "lilconfig": "^3.1.0",
     "lowdb": "^7.0.1",
     "mime": "^4.0.1",
+    "nanoid": "^5.0.5",
     "nostr-tools": "^2.2.1",
+    "socks-proxy-agent": "^8.0.2",
     "websocket-polyfill": "^0.0.3",
     "yaml": "^2.3.4"
   },

diff --git a/public/index.html b/public/index.html
@@ -7,9 +7,45 @@
   </head>
   <body>
     <h1>blobstr</h1>
-    <p>A http file server with a nostr twist</p>
+    <p>Generic blob storage and retrieval for nostr</p>
 
     <h2>Usage:</h2>
-    <code>https://localhost:3000/b91dfd7565edb9382a5537b80bd5f24c6cbe9aee693abcdb7448098f1e8c608b.png</code>
+    <code>GET /b91dfd7565edb9382a5537b80bd5f24c6cbe9aee693abcdb7448098f1e8c608b.png</code>
+    <a href="/b91dfd7565edb9382a5537b80bd5f24c6cbe9aee693abcdb7448098f1e8c608b.png">go</a>
+
+    <h2>List:</h2>
+    <code>GET /list</code>
+    <a href="/list">go</a>
+    <br />
+    <p>Optionally with a pubkey</p>
+    <code>GET /list?pubkey=266815e0c9210dfa324c6cba3573b14bee49da4209a9456f9484e5106cd408a5</code>
+
+    <h2>Upload:</h2>
+    <code>PUT /item</code>
+    <br />
+    <br />
+    <input type="file" id="file" />
+    <button id="upload">Upload</button>
+
+    <script>
+      const input = document.getElementById("file");
+      const button = document.getElementById("upload");
+
+      button.addEventListener("click", async () => {
+        const file = input.files[0];
+        if (!file) return;
+
+        button.textContent = "Uploading...";
+        await fetch("/item", { method: "PUT", body: file }).then(async (res) => {
+          if (res.ok) {
+            const body = await res.json();
+
+            input.value = "";
+            window.open(body.url);
+          } else alert(await res.text());
+        });
+        button.textContent = "Upload";
+      });
+    </script>
   </body>
 </html>
diff --git a/src/config.ts b/src/config.ts
@@ -1,8 +1,9 @@
 import { lilconfig } from "lilconfig";
 import yaml from "yaml";
 
-export type Rule = { type?: string; expiration: string; pubkeys?: string[] };
+export type Rule = { type?: string; pubkeys?: string[]; expiration: string };
 export type Config = {
+  publicDomain: string;
   discovery: {
     nostr: {
       enabled: boolean;
@@ -17,6 +18,14 @@ export type Config = {
     dir: string;
     rules: Rule[];
   };
+  upload: {
+    enabled: boolean;
+    rules: Rule[];
+  };
+  tor: {
+    enabled: boolean;
+    proxy: string;
+  };
 };
 
 function loadYaml(filepath: string, content: string) {

diff --git a/src/db.ts b/src/db.ts
@@ -0,0 +1,49 @@
+import dayjs from "dayjs";
+import { JSONFilePreset } from "lowdb/node";
+import path from "node:path";
+
+type DBSchema = {
+  blobs: Record<string, { expiration?: number; pubkeys?: string[]; created: number; mimeType?: string; size?: number }>;
+};
+const db = await JSONFilePreset<DBSchema>(path.join(process.cwd(), "database.json"), {
+  blobs: {},
+});
+db.data.blobs = db.data.blobs || {};
+setInterval(() => db.write(), 1000);
+
+export function hasBlobEntry(hash: string) {
+  return !!db.data.blobs[hash];
+}
+export function getOrCreateBlobEntry(hash: string) {
+  let blob = db.data.blobs[hash];
+  if (!blob) blob = db.data.blobs[hash] = { created: dayjs().unix() };
+  return blob;
+}
+export function setBlobExpiration(hash: string, expiration: number) {
+  let blob = getOrCreateBlobEntry(hash);
+
+  if (blob.expiration) blob.expiration = Math.max(blob.expiration, expiration);
+  else blob.expiration = expiration;
+}
+export function setBlobMimetype(hash: string, mimeType: string) {
+  let blob = getOrCreateBlobEntry(hash);
+  blob.mimeType = mimeType;
+}
+export function setBlobSize(hash: string, size: number) {
+  let blob = getOrCreateBlobEntry(hash);
+  blob.size = size;
+}
+export function addPubkeyToBlob(hash: string, pubkey: string) {
+  let blob = getOrCreateBlobEntry(hash);
+  if (blob.pubkeys) {
+    if (!blob.pubkeys.includes(pubkey)) blob.pubkeys.push(pubkey);
+  } else blob.pubkeys = [pubkey];
+}
+export function removePubkeyFromBlob(hash: string, pubkey: string) {
+  let blob = getOrCreateBlobEntry(hash);
+  if (blob.pubkeys) {
+    if (blob.pubkeys.includes(pubkey)) blob.pubkeys.splice(blob.pubkeys.indexOf(pubkey), 1);
+  }
+}
+
+export { db };
diff --git a/src/index.ts b/src/index.ts
@@ -5,15 +5,19 @@ import debug from "debug";
 import serve from "koa-static";
 import path from "node:path";
 import { PassThrough } from "node:stream";
+import { URLSearchParams } from "node:url";
+import mime from "mime";
+import pfs from "node:fs/promises";
 
+import { config } from "./config.js";
+import { BlobPointer, BlobSearch } from "./types.js";
 import * as fileStorage from "./storage/file.js";
 import * as cdnDiscovery from "./discover/cdn.js";
 import * as nostrDiscovery from "./discover/nostr.js";
 import * as httpTransport from "./transport/http.js";
-import { config } from "./config.js";
-import { BlobPointer, BlobSearch } from "./types.js";
-import { URLSearchParams } from "node:url";
-import mime from "mime";
+import * as uploadModule from "./storage/upload.js";
+import { db, setBlobExpiration, setBlobMimetype, setBlobSize } from "./db.js";
+import { getExpirationTime, getFileRule } from "./storage/rules.js";
 
 const log = debug("cdn");
 const app = new Koa();
@@ -36,7 +40,15 @@ async function handlePointers(ctx: Koa.ParameterizedContext, pointers: BlobPoint
   return false;
 }
 
+function getBlobURL(hash: string) {
+  const mimeType = db.data.blobs[hash]?.mimeType;
+  const ext = mimeType && mime.getExtension(mimeType);
+  return new URL(hash + (ext ? "." + ext : ""), config.publicDomain).toString();
+}
+
+// fetch blobs
 app.use(async (ctx, next) => {
+  if (ctx.method !== "GET") return next();
   const match = ctx.path.match(/([0-9a-f]{64})(\.[a-z]+)?/);
   if (!match) return next();
 
@@ -84,8 +96,87 @@ app.use(async (ctx, next) => {
   }
 });
 
+// upload blobs
+app.use(async (ctx, next) => {
+  if (ctx.path !== "/item" && ctx.method !== "PUT") return next();
+  if (!config.upload.enabled) {
+    ctx.status = 403;
+    ctx.body = "Uploads disabled";
+    return;
+  }
+
+  // handle upload
+  try {
+    const auth = ctx.query.auth as string | undefined;
+    const contentType = ctx.header["content-type"];
+
+    const rule = getFileRule(
+      {
+        mimeType: contentType,
+        // pubkey: metadata?.pubkey,
+      },
+      config.upload.rules,
+    );
+    if (!rule) {
+      ctx.status = 403;
+      return;
+    }
+
+    const metadata = await uploadModule.uploadWriteStream(ctx.req);
+    const mimeType = contentType || metadata.mimeType;
+
+    // save the file if its not already there
+    if (!db.data.blobs[metadata.hash]) {
+      setBlobSize(metadata.hash, metadata.size);
+      setBlobExpiration(metadata.hash, getExpirationTime(rule));
+      if (mimeType) setBlobMimetype(metadata.hash, mimeType);
+
+      await fileStorage.saveTempFile(metadata.hash, metadata.tempFile, mimeType);
+    } else {
+      await pfs.rm(metadata.tempFile);
+    }
+
+    ctx.status = 200;
+    ctx.body = {
+      url: getBlobURL(metadata.hash),
+      sha256: metadata.hash,
+      type: mimeType,
+    };
+  } catch (e) {
+    ctx.status = 403;
+    if (e instanceof Error) ctx.body = e.message;
+  }
+});
+
+// list blobs
+app.use(async (ctx, next) => {
+  if (ctx.method !== "GET" || ctx.path !== "/list") return next();
+
+  const filter = ctx.query as { pubkey?: string };
+
+  ctx.status = 200;
+  ctx.body = Object.entries(db.data.blobs)
+    .filter(([hash, blob]) => (filter.pubkey ? blob.pubkeys?.includes(filter.pubkey) : true))
+    .map(([hash, blob]) => ({
+      sha256: hash,
+      created: blob.created,
+      url: getBlobURL(hash),
+      type: blob.mimeType,
+      size: blob.size,
+    }));
+});
+
 app.use(serve(path.join(process.cwd(), "public")));
 
 app.listen(3000);
 
 setInterval(() => fileStorage.prune(), 1000 * 30);
+
+async function shutdown() {
+  log("Saving database...");
+  await db.write();
+  process.exit(0);
+}
+
+process.addListener("SIGTERM", shutdown);
+process.addListener("SIGINT", shutdown);
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ data @@
     .env
     node_modules
     .github
+    database.json