feat: support specifying npm registries via .npmrc (#427)

* chore(sb_module_loader): add a dependency

* chore: update `Cargo.lock`

* feat: support specifying npm registries via .npmrc

* chore: add examples related to the private npm registry import

* chore: update examples

* stamp: polishing

* chore: add an integration test

* stamp: polishing

* stamp: oops

* stamp: meow

* fix: find .npmrc by traversing parent directories

* chore: add an example

* fix: make npm resolver respect `noModuleCache` option

* fix: add `--disable-module-cache` flag to bundle command

* chore: update an integration test and test cases

crates/base/src/deno_runtime.rs

@@ -1,6 +1,7 @@
 use crate::inspector_server::Inspector;
 use crate::rt_worker::supervisor::{CPUUsage, CPUUsageMetrics};
 use crate::rt_worker::worker::DuplexStreamEntry;
+use crate::utils::path::find_up;
 use crate::utils::units::{bytes_to_display, mib_to_bytes};
 
 use anyhow::{anyhow, bail, Context, Error};
@@ -333,8 +334,15 @@ where
                 CacheSetting::Use
             };
 
+            if let Some(npmrc_path) = find_up(".npmrc", &base_dir_path) {
+                if npmrc_path.exists() && npmrc_path.is_file() {
+                    emitter_factory.set_npmrc_path(npmrc_path);
+                    emitter_factory.set_npmrc_env_vars(env_vars.clone());
+                }
+            }
+
             emitter_factory.set_file_fetcher_allow_remote(allow_remote_modules);
-            emitter_factory.set_file_fetcher_cache_strategy(cache_strategy);
+            emitter_factory.set_cache_strategy(cache_strategy);
             emitter_factory.set_decorator_type(maybe_decorator);
 
             if let Some(jsx_import_source_config) = maybe_jsx_import_source_config.clone() {

crates/base/src/lib.rs

@@ -12,6 +12,7 @@ mod inspector_server;
 mod timeout;
 
 pub use inspector_server::InspectorOption;
+pub use sb_core::cache::CacheSetting;
 pub use sb_graph::DecoratorType;
 
 #[cfg(test)]

crates/base/src/rt_worker/utils.rs

@@ -1,6 +1,6 @@
-use event_worker::events::{EventMetadata, WorkerEventWithMetadata};
+use event_worker::events::{EventMetadata, WorkerEventWithMetadata, WorkerEvents};
 use sb_workers::context::{UserWorkerMsgs, WorkerRuntimeOpts};
-use tokio::sync::mpsc::UnboundedSender;
+use tokio::sync::mpsc::{self, UnboundedSender};
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
@@ -46,3 +46,13 @@ pub fn get_event_metadata(conf: &WorkerRuntimeOpts) -> EventMetadata {
 
     event_metadata
 }
+
+pub fn send_event_if_event_worker_available(
+    maybe_event_worker: Option<&mpsc::UnboundedSender<WorkerEventWithMetadata>>,
+    event: WorkerEvents,
+    metadata: EventMetadata,
+) {
+    if let Some(event_worker) = maybe_event_worker {
+        let _ = event_worker.send(WorkerEventWithMetadata { event, metadata });
+    }
+}

crates/base/src/rt_worker/worker.rs

@@ -1,9 +1,11 @@
 use crate::deno_runtime::DenoRuntime;
 use crate::inspector_server::Inspector;
 use crate::rt_worker::supervisor;
-use crate::rt_worker::utils::{get_event_metadata, parse_worker_conf};
+use crate::rt_worker::utils::{
+    get_event_metadata, parse_worker_conf, send_event_if_event_worker_available,
+};
 use crate::rt_worker::worker_ctx::create_supervisor;
-use crate::utils::send_event_if_event_worker_available;
+
 use anyhow::Error;
 use base_mem_check::MemCheckState;
 use base_rt::error::CloneableError;

crates/base/src/rt_worker/worker_ctx.rs

@@ -1,8 +1,8 @@
+use super::utils::send_event_if_event_worker_available;
 use crate::deno_runtime::DenoRuntime;
 use crate::inspector_server::Inspector;
 use crate::server::ServerFlags;
 use crate::timeout::{self, CancelOnWriteTimeout, ReadTimeoutStream};
-use crate::utils::send_event_if_event_worker_available;
 
 use crate::rt_worker::worker::{Worker, WorkerHandler};
 use crate::rt_worker::worker_pool::WorkerPool;

crates/base/src/server.rs

@@ -6,6 +6,7 @@ use crate::rt_worker::worker_pool::WorkerPoolPolicy;
 use crate::InspectorOption;
 use anyhow::{anyhow, bail, Context, Error};
 use deno_config::JsxImportSourceConfig;
+use enum_as_inner::EnumAsInner;
 use futures_util::future::{poll_fn, BoxFuture};
 use futures_util::{FutureExt, Stream};
 use hyper_v014::{server::conn::Http, service::Service, Body, Request, Response};
@@ -52,6 +53,7 @@ pub enum ServerEvent {
     Draining,
 }
 
+#[derive(Debug, EnumAsInner)]
 pub enum ServerHealth {
     Listening(mpsc::UnboundedReceiver<ServerEvent>, SharedMetricSource),
     Failure,
@@ -233,6 +235,7 @@ impl Service<Request<Body>> for WorkerService {
     }
 }
 
+#[derive(Default)]
 pub struct WorkerEntrypoints {
     pub main: Option<String>,
     pub events: Option<String>,

crates/base/src/utils/mod.rs

@@ -0,0 +1,2 @@
+pub mod path;
+pub mod units;

crates/base/src/utils/path.rs

@@ -0,0 +1,23 @@
+use std::path::{Path, PathBuf};
+
+pub fn find_up<A, B>(filename: A, start: B) -> Option<PathBuf>
+where
+    A: AsRef<Path>,
+    B: AsRef<Path>,
+{
+    let mut current = start.as_ref().to_path_buf();
+
+    loop {
+        if current.join(&filename).exists() {
+            return Some(current.join(filename));
+        }
+
+        if let Some(parent) = current.parent() {
+            current = parent.to_path_buf();
+        } else {
+            break;
+        }
+    }
+
+    None
+}

crates/base/test_cases/main_eszip/index.ts

@@ -24,7 +24,7 @@ Deno.serve(async (req: Request) => {
     const workerTimeoutMs = 10 * 60 * 1000;
     const cpuTimeSoftLimitMs = 10 * 60 * 1000;
     const cpuTimeHardLimitMs = 10 * 60 * 1000;
-    const noModuleCache = false;
+    const noModuleCache = true;
     const importMapPath = null;
     const envVarsObj = Deno.env.toObject();
     const envVars = Object.keys(envVarsObj).map(k => [k, envVarsObj[k]]);

crates/base/test_cases/main_with_registry/index.ts

@@ -0,0 +1,74 @@
+import { handleRegistryRequest } from "./registry/mod.ts";
+
+console.log('main function started');
+
+Deno.serve(async (req: Request) => {
+  console.log(req.url);
+  const url = new URL(req.url);
+  const { pathname } = url;
+  const path_parts = pathname.split("/");
+  let service_name = path_parts[1];
+
+  if (req.headers.has('x-service-path')) {
+    service_name = req.headers.get('x-service-path')!;
+  }
+
+  const REGISTRY_PREFIX = '/_internal/registry';
+  if (pathname.startsWith(REGISTRY_PREFIX)) {
+    return await handleRegistryRequest(REGISTRY_PREFIX, req);
+  }
+
+  if (!service_name || service_name === "") {
+    const error = { msg: "missing function name in request" }
+    return new Response(
+      JSON.stringify(error),
+      { status: 400, headers: { "Content-Type": "application/json" } },
+    )
+  }
+
+  const servicePath = `./test_cases/${service_name}`;
+  console.error(`serving the request with ${servicePath}`);
+
+  const createWorker = async () => {
+    const memoryLimitMb = 150;
+    const workerTimeoutMs = 10 * 60 * 1000;
+    const cpuTimeSoftLimitMs = 10 * 60 * 1000;
+    const cpuTimeHardLimitMs = 10 * 60 * 1000;
+    const noModuleCache = true;
+    const importMapPath = null;
+    const envVarsObj = Deno.env.toObject();
+    const envVars = Object.keys(envVarsObj).map(k => [k, envVarsObj[k]]);
+
+    return await EdgeRuntime.userWorkers.create({
+      servicePath,
+      memoryLimitMb,
+      workerTimeoutMs,
+      cpuTimeSoftLimitMs,
+      cpuTimeHardLimitMs,
+      noModuleCache,
+      importMapPath,
+      envVars
+    });
+  }
+
+  const callWorker = async () => {
+    try {
+      const worker = await createWorker();
+      return await worker.fetch(req);
+    } catch (e) {
+      console.error(e);
+
+      // if (e instanceof Deno.errors.WorkerRequestCancelled) {
+      // 	return await callWorker();
+      // }
+
+      const error = { msg: e.toString() }
+      return new Response(
+        JSON.stringify(error),
+        { status: 500, headers: { "Content-Type": "application/json" } },
+      );
+    }
+  }
+
+  return callWorker();
+})

crates/base/test_cases/main_with_registry/registry/mock.ts

@@ -0,0 +1 @@
+export default "H4sIAMaFtVcAA+2Vz2rDMAzGc85TiJw2GKmT2g6MbfQ02GG3vYDnuG2axjZ2Mgpl7z7nXzcKXS/Nxph/F4Ek8ikR+qIZL9lKzIIJQQhlhICLSUbQ13gAktS1zFFKKQKUzFNKAiBTDjXS2JoZN4qwWsj16b5z9eE9DvGPoIf9FzIXu3hjp9Bw34NS+s3+8ef+cbd/nKU0ADTFMMf88/1XKm+2IhY7rUxt4R6WjeR1oeTVNezfw/C35/NMy3j/Q3QOoOSlNdr7x/j0/acEH/k/IQT7+/8J9iFAJFkloluIFpXipeVKi9lSqVdmopu2/CaMdY7QdiQxilGfzYXlptD1UHlkpYDeTfp6xYquMP5Z+uy24ELaTu356aXPsaZeK9M9pFOFu159IXW1sbEyq4co9F7k8Xg8F+UDnayJaQAOAAA=";

crates/base/test_cases/main_with_registry/registry/mod.ts

@@ -0,0 +1,3 @@
+import registryHandler from './registry-handler.ts';
+
+export { registryHandler as handleRegistryRequest };

crates/base/test_cases/main_with_registry/registry/registry-handler.ts

@@ -0,0 +1,81 @@
+import crypto from "node:crypto";
+import assign from "npm:object-assign";
+
+import mock from "./mock.ts";
+
+const PKG_NAME = "@meowmeow/foobar";
+const TOKEN = "MeowMeowToken";
+const RESPONSE_TEMPLATE = await import("./template.json");
+
+export default async function (prefix: string, req: Request) {
+    if (req.method !== "GET") {
+        return Response.json({ error: "Method Not Allowed" }, { status: 405 });
+    }
+
+    if (!req.headers.get("authorization")) {
+        return Response.json({ error: "Missing authorization header" }, { status: 403 });
+    }
+
+    const token = req.headers.get("authorization")?.split(" ", 2);
+    const isValidTokenType = token?.[0] === "Bearer";
+    const isValidToken = token?.[1] === TOKEN;
+
+    if (!isValidTokenType || !isValidToken) {
+        return Response.json({ error: "Incorrect token" }, { status: 403 });
+    }
+
+
+    const url = new URL(req.url);
+    const { pathname } = url;
+
+    const moduleName = PKG_NAME.split("/", 2)[1];
+    const buf = new Uint8Array(await ((await fetch(`data:application/octet;base64,${mock}`)).arrayBuffer()));
+    const sum = sha1(buf);
+
+    if (ucEnc(pathname) === `${prefix}/${softEncode(PKG_NAME)}` || ucEnc(pathname) === `${prefix}/${PKG_NAME}`) {
+        return Response.json(generatePackageResponse("localhost", prefix, moduleName, sum));
+    }
+
+
+    if (pathname === `${prefix}/${PKG_NAME}/-/${moduleName}-1.0.0.tgz`) {
+        return new Response(buf);
+    }
+
+    return Response.json({ error: "File Not Found" }, { status: 404 });
+}
+
+function ucEnc(str: string) {
+    return str.replace(/(%[a-f0-9]{2})/g, function (match) {
+        return match.toUpperCase()
+    })
+}
+
+function softEncode(pkg: string) {
+    return encodeURIComponent(pkg).replace(/^%40/, '@')
+}
+
+function generatePackageResponse(hostname: string, prefix: string, moduleName: string, sum: string) {
+    const port = Deno.env.get("EDGE_RUNTIME_PORT") || "9998";
+    const tpl = assign({}, RESPONSE_TEMPLATE.default, {
+        _id: PKG_NAME,
+        name: PKG_NAME,
+    });
+
+    tpl.versions["1.0.0"] = assign({}, tpl.versions["1.0.0"], {
+        _id: PKG_NAME + "@1.0.0",
+        name: PKG_NAME,
+        dist: assign({}, tpl.versions["1.0.0"].dist, {
+            shasum: sum,
+            tarball: [
+                "http://" + hostname + ":" + port + prefix,
+                PKG_NAME, "-", moduleName + "-1.0.0.tgz"
+            ].join("/")
+        })
+    })
+
+    return tpl
+}
+
+function sha1(data: Uint8Array) {
+    return crypto.createHash("sha1").update(data).digest('hex')
+}

crates/base/test_cases/main_with_registry/registry/template.json

@@ -0,0 +1,49 @@
+{
+    "_id": "@mockscope/foobar",
+    "_rev": "1-c30105564f195a3038d3348840c9e080",
+    "name": "@mockscope/foobar",
+    "description": "Fake module",
+    "dist-tags": {
+      "latest": "1.0.0"
+    },
+    "versions": {
+      "1.0.0": {
+        "name": "@mockscope/foobar",
+        "version": "1.0.0",
+        "description": "Fake module",
+        "main": "index.js",
+        "license": "MIT",
+        "gitHead": "2666d6874aaebbaf7188e14d94eb8488079d6c2c",
+        "_id": "@mockscope/[email protected]",
+        "_shasum": "19d13344e0b701cae3374fa6f03d2d3f90847c1c",
+        "_from": ".",
+        "_npmVersion": "2.5.1",
+        "_nodeVersion": "1.2.0",
+        "_npmUser": {
+          "name": "foobar",
+          "email": "[email protected]"
+        },
+        "maintainers": [{
+          "name": "foobar",
+          "email": "[email protected]"
+        }],
+        "dist": {
+          "shasum": "cd2e97011c99721c5f0a6d677c50a144ec790a2d"
+        },
+        "directories": {}
+      }
+    },
+    "readme": "Mock module!\n",
+    "maintainers": [{
+      "name": "foobar",
+      "email": "[email protected]"
+    }],
+    "time": {
+      "modified": "2015-05-05T10:32:06.793Z",
+      "created": "2015-05-05T10:32:06.793Z",
+      "1.0.0": "2015-05-05T10:32:06.793Z"
+    },
+    "license": "MIT",
+    "readmeFilename": "README.md",
+    "_attachments": {}
+  }

crates/base/test_cases/private-npm-package-import-2/.npmrc

@@ -0,0 +1,2 @@
+@meowmeow:registry=http://localhost:8498/_internal/registry/
+//localhost:8498/_internal/registry/:_authToken=MeowMeowToken

crates/base/test_cases/private-npm-package-import-2/inner/index.js

@@ -0,0 +1,13 @@
+import foobar from "npm:@meowmeow/foobar";
+import isOdd from "npm:is-odd";
+
+console.log(foobar());
+
+export default {
+    fetch() {
+        return Response.json({
+            meow: typeof foobar,
+            odd: isOdd(1),
+        });
+    }
+}