Skip to content

Commit

Permalink
feat: support specifying npm registries via .npmrc (#427)
Browse files Browse the repository at this point in the history
* chore(sb_module_loader): add a dependency

* chore: update `Cargo.lock`

* feat: support specifying npm registries via .npmrc

* chore: add examples related to the private npm registry import

* chore: update examples

* stamp: polishing

* chore: add an integration test

* stamp: polishing

* stamp: oops

* stamp: meow

* fix: find .npmrc by traversing parent directories

* chore: add an example

* fix: make npm resolver respect `noModuleCache` option

* fix: add `--disable-module-cache` flag to bundle command

* chore: update an integration test and test cases
  • Loading branch information
nyannyacha authored Oct 17, 2024
1 parent fbddb16 commit 051a276
Show file tree
Hide file tree
Showing 40 changed files with 795 additions and 44 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 9 additions & 1 deletion crates/base/src/deno_runtime.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use crate::inspector_server::Inspector;
use crate::rt_worker::supervisor::{CPUUsage, CPUUsageMetrics};
use crate::rt_worker::worker::DuplexStreamEntry;
use crate::utils::path::find_up;
use crate::utils::units::{bytes_to_display, mib_to_bytes};

use anyhow::{anyhow, bail, Context, Error};
Expand Down Expand Up @@ -333,8 +334,15 @@ where
CacheSetting::Use
};

if let Some(npmrc_path) = find_up(".npmrc", &base_dir_path) {
if npmrc_path.exists() && npmrc_path.is_file() {
emitter_factory.set_npmrc_path(npmrc_path);
emitter_factory.set_npmrc_env_vars(env_vars.clone());
}
}

emitter_factory.set_file_fetcher_allow_remote(allow_remote_modules);
emitter_factory.set_file_fetcher_cache_strategy(cache_strategy);
emitter_factory.set_cache_strategy(cache_strategy);
emitter_factory.set_decorator_type(maybe_decorator);

if let Some(jsx_import_source_config) = maybe_jsx_import_source_config.clone() {
Expand Down
1 change: 1 addition & 0 deletions crates/base/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ mod inspector_server;
mod timeout;

pub use inspector_server::InspectorOption;
pub use sb_core::cache::CacheSetting;
pub use sb_graph::DecoratorType;

#[cfg(test)]
Expand Down
14 changes: 12 additions & 2 deletions crates/base/src/rt_worker/utils.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use event_worker::events::{EventMetadata, WorkerEventWithMetadata};
use event_worker::events::{EventMetadata, WorkerEventWithMetadata, WorkerEvents};
use sb_workers::context::{UserWorkerMsgs, WorkerRuntimeOpts};
use tokio::sync::mpsc::UnboundedSender;
use tokio::sync::mpsc::{self, UnboundedSender};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;

Expand Down Expand Up @@ -46,3 +46,13 @@ pub fn get_event_metadata(conf: &WorkerRuntimeOpts) -> EventMetadata {

event_metadata
}

pub fn send_event_if_event_worker_available(
maybe_event_worker: Option<&mpsc::UnboundedSender<WorkerEventWithMetadata>>,
event: WorkerEvents,
metadata: EventMetadata,
) {
if let Some(event_worker) = maybe_event_worker {
let _ = event_worker.send(WorkerEventWithMetadata { event, metadata });
}
}
6 changes: 4 additions & 2 deletions crates/base/src/rt_worker/worker.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
use crate::deno_runtime::DenoRuntime;
use crate::inspector_server::Inspector;
use crate::rt_worker::supervisor;
use crate::rt_worker::utils::{get_event_metadata, parse_worker_conf};
use crate::rt_worker::utils::{
get_event_metadata, parse_worker_conf, send_event_if_event_worker_available,
};
use crate::rt_worker::worker_ctx::create_supervisor;
use crate::utils::send_event_if_event_worker_available;

use anyhow::Error;
use base_mem_check::MemCheckState;
use base_rt::error::CloneableError;
Expand Down
2 changes: 1 addition & 1 deletion crates/base/src/rt_worker/worker_ctx.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use super::utils::send_event_if_event_worker_available;
use crate::deno_runtime::DenoRuntime;
use crate::inspector_server::Inspector;
use crate::server::ServerFlags;
use crate::timeout::{self, CancelOnWriteTimeout, ReadTimeoutStream};
use crate::utils::send_event_if_event_worker_available;

use crate::rt_worker::worker::{Worker, WorkerHandler};
use crate::rt_worker::worker_pool::WorkerPool;
Expand Down
3 changes: 3 additions & 0 deletions crates/base/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use crate::rt_worker::worker_pool::WorkerPoolPolicy;
use crate::InspectorOption;
use anyhow::{anyhow, bail, Context, Error};
use deno_config::JsxImportSourceConfig;
use enum_as_inner::EnumAsInner;
use futures_util::future::{poll_fn, BoxFuture};
use futures_util::{FutureExt, Stream};
use hyper_v014::{server::conn::Http, service::Service, Body, Request, Response};
Expand Down Expand Up @@ -52,6 +53,7 @@ pub enum ServerEvent {
Draining,
}

#[derive(Debug, EnumAsInner)]
pub enum ServerHealth {
Listening(mpsc::UnboundedReceiver<ServerEvent>, SharedMetricSource),
Failure,
Expand Down Expand Up @@ -233,6 +235,7 @@ impl Service<Request<Body>> for WorkerService {
}
}

#[derive(Default)]
pub struct WorkerEntrypoints {
pub main: Option<String>,
pub events: Option<String>,
Expand Down
14 changes: 0 additions & 14 deletions crates/base/src/utils.rs

This file was deleted.

2 changes: 2 additions & 0 deletions crates/base/src/utils/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pub mod path;
pub mod units;
23 changes: 23 additions & 0 deletions crates/base/src/utils/path.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
use std::path::{Path, PathBuf};

pub fn find_up<A, B>(filename: A, start: B) -> Option<PathBuf>
where
A: AsRef<Path>,
B: AsRef<Path>,
{
let mut current = start.as_ref().to_path_buf();

loop {
if current.join(&filename).exists() {
return Some(current.join(filename));
}

if let Some(parent) = current.parent() {
current = parent.to_path_buf();
} else {
break;
}
}

None
}
2 changes: 1 addition & 1 deletion crates/base/test_cases/main_eszip/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ Deno.serve(async (req: Request) => {
const workerTimeoutMs = 10 * 60 * 1000;
const cpuTimeSoftLimitMs = 10 * 60 * 1000;
const cpuTimeHardLimitMs = 10 * 60 * 1000;
const noModuleCache = false;
const noModuleCache = true;
const importMapPath = null;
const envVarsObj = Deno.env.toObject();
const envVars = Object.keys(envVarsObj).map(k => [k, envVarsObj[k]]);
Expand Down
74 changes: 74 additions & 0 deletions crates/base/test_cases/main_with_registry/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { handleRegistryRequest } from "./registry/mod.ts";

console.log('main function started');

Deno.serve(async (req: Request) => {
console.log(req.url);
const url = new URL(req.url);
const { pathname } = url;
const path_parts = pathname.split("/");
let service_name = path_parts[1];

if (req.headers.has('x-service-path')) {
service_name = req.headers.get('x-service-path')!;
}

const REGISTRY_PREFIX = '/_internal/registry';
if (pathname.startsWith(REGISTRY_PREFIX)) {
return await handleRegistryRequest(REGISTRY_PREFIX, req);
}

if (!service_name || service_name === "") {
const error = { msg: "missing function name in request" }
return new Response(
JSON.stringify(error),
{ status: 400, headers: { "Content-Type": "application/json" } },
)
}

const servicePath = `./test_cases/${service_name}`;
console.error(`serving the request with ${servicePath}`);

const createWorker = async () => {
const memoryLimitMb = 150;
const workerTimeoutMs = 10 * 60 * 1000;
const cpuTimeSoftLimitMs = 10 * 60 * 1000;
const cpuTimeHardLimitMs = 10 * 60 * 1000;
const noModuleCache = true;
const importMapPath = null;
const envVarsObj = Deno.env.toObject();
const envVars = Object.keys(envVarsObj).map(k => [k, envVarsObj[k]]);

return await EdgeRuntime.userWorkers.create({
servicePath,
memoryLimitMb,
workerTimeoutMs,
cpuTimeSoftLimitMs,
cpuTimeHardLimitMs,
noModuleCache,
importMapPath,
envVars
});
}

const callWorker = async () => {
try {
const worker = await createWorker();
return await worker.fetch(req);
} catch (e) {
console.error(e);

// if (e instanceof Deno.errors.WorkerRequestCancelled) {
// return await callWorker();
// }

const error = { msg: e.toString() }
return new Response(
JSON.stringify(error),
{ status: 500, headers: { "Content-Type": "application/json" } },
);
}
}

return callWorker();
})
1 change: 1 addition & 0 deletions crates/base/test_cases/main_with_registry/registry/mock.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export default "H4sIAMaFtVcAA+2Vz2rDMAzGc85TiJw2GKmT2g6MbfQ02GG3vYDnuG2axjZ2Mgpl7z7nXzcKXS/Nxph/F4Ek8ikR+qIZL9lKzIIJQQhlhICLSUbQ13gAktS1zFFKKQKUzFNKAiBTDjXS2JoZN4qwWsj16b5z9eE9DvGPoIf9FzIXu3hjp9Bw34NS+s3+8ef+cbd/nKU0ADTFMMf88/1XKm+2IhY7rUxt4R6WjeR1oeTVNezfw/C35/NMy3j/Q3QOoOSlNdr7x/j0/acEH/k/IQT7+/8J9iFAJFkloluIFpXipeVKi9lSqVdmopu2/CaMdY7QdiQxilGfzYXlptD1UHlkpYDeTfp6xYquMP5Z+uy24ELaTu356aXPsaZeK9M9pFOFu159IXW1sbEyq4co9F7k8Xg8F+UDnayJaQAOAAA=";
3 changes: 3 additions & 0 deletions crates/base/test_cases/main_with_registry/registry/mod.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import registryHandler from './registry-handler.ts';

export { registryHandler as handleRegistryRequest };
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import crypto from "node:crypto";
import assign from "npm:object-assign";

import mock from "./mock.ts";

const PKG_NAME = "@meowmeow/foobar";
const TOKEN = "MeowMeowToken";
const RESPONSE_TEMPLATE = await import("./template.json");

export default async function (prefix: string, req: Request) {
if (req.method !== "GET") {
return Response.json({ error: "Method Not Allowed" }, { status: 405 });
}

if (!req.headers.get("authorization")) {
return Response.json({ error: "Missing authorization header" }, { status: 403 });
}

const token = req.headers.get("authorization")?.split(" ", 2);
const isValidTokenType = token?.[0] === "Bearer";
const isValidToken = token?.[1] === TOKEN;

if (!isValidTokenType || !isValidToken) {
return Response.json({ error: "Incorrect token" }, { status: 403 });
}


const url = new URL(req.url);
const { pathname } = url;

const moduleName = PKG_NAME.split("/", 2)[1];
const buf = new Uint8Array(await ((await fetch(`data:application/octet;base64,${mock}`)).arrayBuffer()));
const sum = sha1(buf);

if (ucEnc(pathname) === `${prefix}/${softEncode(PKG_NAME)}` || ucEnc(pathname) === `${prefix}/${PKG_NAME}`) {
return Response.json(generatePackageResponse("localhost", prefix, moduleName, sum));
}


if (pathname === `${prefix}/${PKG_NAME}/-/${moduleName}-1.0.0.tgz`) {
return new Response(buf);
}

return Response.json({ error: "File Not Found" }, { status: 404 });
}

function ucEnc(str: string) {
return str.replace(/(%[a-f0-9]{2})/g, function (match) {
return match.toUpperCase()
})
}

function softEncode(pkg: string) {
return encodeURIComponent(pkg).replace(/^%40/, '@')
}

function generatePackageResponse(hostname: string, prefix: string, moduleName: string, sum: string) {
const port = Deno.env.get("EDGE_RUNTIME_PORT") || "9998";
const tpl = assign({}, RESPONSE_TEMPLATE.default, {
_id: PKG_NAME,
name: PKG_NAME,
});

tpl.versions["1.0.0"] = assign({}, tpl.versions["1.0.0"], {
_id: PKG_NAME + "@1.0.0",
name: PKG_NAME,
dist: assign({}, tpl.versions["1.0.0"].dist, {
shasum: sum,
tarball: [
"http://" + hostname + ":" + port + prefix,
PKG_NAME, "-", moduleName + "-1.0.0.tgz"
].join("/")
})
})

return tpl
}

function sha1(data: Uint8Array) {
return crypto.createHash("sha1").update(data).digest('hex')
}
49 changes: 49 additions & 0 deletions crates/base/test_cases/main_with_registry/registry/template.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
"_id": "@mockscope/foobar",
"_rev": "1-c30105564f195a3038d3348840c9e080",
"name": "@mockscope/foobar",
"description": "Fake module",
"dist-tags": {
"latest": "1.0.0"
},
"versions": {
"1.0.0": {
"name": "@mockscope/foobar",
"version": "1.0.0",
"description": "Fake module",
"main": "index.js",
"license": "MIT",
"gitHead": "2666d6874aaebbaf7188e14d94eb8488079d6c2c",
"_id": "@mockscope/[email protected]",
"_shasum": "19d13344e0b701cae3374fa6f03d2d3f90847c1c",
"_from": ".",
"_npmVersion": "2.5.1",
"_nodeVersion": "1.2.0",
"_npmUser": {
"name": "foobar",
"email": "[email protected]"
},
"maintainers": [{
"name": "foobar",
"email": "[email protected]"
}],
"dist": {
"shasum": "cd2e97011c99721c5f0a6d677c50a144ec790a2d"
},
"directories": {}
}
},
"readme": "Mock module!\n",
"maintainers": [{
"name": "foobar",
"email": "[email protected]"
}],
"time": {
"modified": "2015-05-05T10:32:06.793Z",
"created": "2015-05-05T10:32:06.793Z",
"1.0.0": "2015-05-05T10:32:06.793Z"
},
"license": "MIT",
"readmeFilename": "README.md",
"_attachments": {}
}
2 changes: 2 additions & 0 deletions crates/base/test_cases/private-npm-package-import-2/.npmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@meowmeow:registry=http://localhost:8498/_internal/registry/
//localhost:8498/_internal/registry/:_authToken=MeowMeowToken
13 changes: 13 additions & 0 deletions crates/base/test_cases/private-npm-package-import-2/inner/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import foobar from "npm:@meowmeow/foobar";
import isOdd from "npm:is-odd";

console.log(foobar());

export default {
fetch() {
return Response.json({
meow: typeof foobar,
odd: isOdd(1),
});
}
}
Loading

0 comments on commit 051a276

Please sign in to comment.