Skip to content

Commit

Permalink
fix: set a module root for loading modules
Browse files Browse the repository at this point in the history
  • Loading branch information
laktek committed Jul 7, 2023
1 parent 5fce6ac commit a03877b
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 98 deletions.
23 changes: 18 additions & 5 deletions crates/base/src/deno_runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use import_map::{parse_from_json, ImportMap, ImportMapDiagnostic};
use log::{debug, warn};
use serde::de::DeserializeOwned;
use std::collections::HashMap;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use std::time::Duration;
use std::{fmt, fs};
Expand Down Expand Up @@ -138,9 +138,8 @@ impl DenoRuntime {
set_v8_flags();

let user_agent = "supabase-edge-runtime".to_string();
let base_url =
Url::from_directory_path(std::env::current_dir().map(|p| p.join(&service_path))?)
.unwrap();
let base_dir_path = std::env::current_dir().map(|p| p.join(&service_path))?;
let base_url = Url::from_directory_path(&base_dir_path).unwrap();
// TODO: check for other potential main paths (eg: index.js, index.tsx)
let main_module_url = base_url.join("index.ts")?;

Expand Down Expand Up @@ -181,7 +180,21 @@ impl DenoRuntime {
];

let import_map = load_import_map(import_map_path)?;
let module_loader = DefaultModuleLoader::new(import_map, no_module_cache)?;
let mut allow_remote_modules = true;
let mut module_root_path = base_dir_path.clone();
if conf.is_user_worker() {
let user_conf = conf.as_user_worker().unwrap();
allow_remote_modules = user_conf.allow_remote_modules;
if let Some(custom_module_root) = &user_conf.custom_module_root {
module_root_path = PathBuf::from(custom_module_root);
}
}
let module_loader = DefaultModuleLoader::new(
module_root_path,
import_map,
no_module_cache,
allow_remote_modules,
)?;

let mut js_runtime = JsRuntime::new(RuntimeOptions {
extensions,
Expand Down
12 changes: 8 additions & 4 deletions crates/base/src/js_worker/module_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use module_fetcher::cache::{
use module_fetcher::emit::emit_parsed_source;
use module_fetcher::file_fetcher::{CacheSetting, FileFetcher};
use module_fetcher::http_util::HttpClient;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use url::Url;

Expand Down Expand Up @@ -53,7 +53,12 @@ pub struct DefaultModuleLoader {
}

impl DefaultModuleLoader {
pub fn new(maybe_import_map: Option<ImportMap>, no_cache: bool) -> Result<Self, AnyError> {
pub fn new(
root_path: PathBuf,
maybe_import_map: Option<ImportMap>,
no_cache: bool,
allow_remote: bool,
) -> Result<Self, AnyError> {
// Note: we are reusing Deno dependency cache path
let deno_dir = DenoDir::new(None)?;
let deps_cache_location = deno_dir.deps_folder_path();
Expand All @@ -64,7 +69,6 @@ impl DefaultModuleLoader {
} else {
CacheSetting::Use
};
let allow_remote = true;
let http_client = make_http_client()?;
let blob_store = deno_web::BlobStore::default();
let file_fetcher = FileFetcher::new(
Expand All @@ -74,7 +78,7 @@ impl DefaultModuleLoader {
http_client,
blob_store,
);
let permissions = module_fetcher::permissions::Permissions::default();
let permissions = module_fetcher::permissions::Permissions::new(root_path);
let emit_cache = EmitCache::new(deno_dir.gen_cache.clone());
let caches_def = caches::Caches::default();
let parsed_source_cache = ParsedSourceCache::new(caches_def.dep_analysis_db(&deno_dir));
Expand Down
25 changes: 21 additions & 4 deletions crates/module_fetcher/src/permissions.rs
Original file line number Diff line number Diff line change
@@ -1,18 +1,32 @@
use deno_core::error::uri_error;
use deno_core::error::AnyError;
use deno_core::ModuleSpecifier;
use std::path::PathBuf;

#[derive(Default, Clone, Debug)]
pub struct Permissions;
pub struct Permissions {
root_path: PathBuf,
}

impl Permissions {
pub fn new(root_path: PathBuf) -> Self {
Self { root_path }
}

/// A helper function that determines if the module specifier is a local or
/// remote, and performs a read or net check for the specifier.
pub fn check_specifier(&mut self, specifier: &ModuleSpecifier) -> Result<(), AnyError> {
match specifier.scheme() {
"file" => match specifier.to_file_path() {
// allow all file paths
Ok(_) => Ok(()),
Ok(file_path) => {
if !file_path.starts_with(&self.root_path) {
return Err(uri_error(format!(
"Invalid file path.\n Specifier: {}",
specifier
)));
}
Ok(())
}
Err(_) => Err(uri_error(format!(
"Invalid file path.\n Specifier: {}",
specifier
Expand All @@ -25,7 +39,10 @@ impl Permissions {
}
}

// TODO: audit the API calling allow_all()
pub fn allow_all() -> Self {
Self
Self {
root_path: PathBuf::new(),
}
}
}
40 changes: 22 additions & 18 deletions crates/sb_worker_context/essentials.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,28 @@ pub struct UserWorkerRuntimeOpts {
pub max_cpu_bursts: u64,

pub force_create: bool,
pub allow_remote_modules: bool,
pub custom_module_root: Option<String>,
}

impl Default for UserWorkerRuntimeOpts {
fn default() -> UserWorkerRuntimeOpts {
UserWorkerRuntimeOpts {
memory_limit_mb: 512,
worker_timeout_ms: 5 * 60 * 1000,
low_memory_multiplier: 5,
max_cpu_bursts: 10,
cpu_burst_interval_ms: 100,
cpu_time_threshold_ms: 50,

force_create: false,
key: None,
pool_msg_tx: None,
events_msg_tx: None,
allow_remote_modules: true,
custom_module_root: None,
}
}
}

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -51,24 +73,6 @@ pub struct WorkerContextInitOpts {
pub conf: WorkerRuntimeOpts,
}

impl Default for UserWorkerRuntimeOpts {
fn default() -> UserWorkerRuntimeOpts {
UserWorkerRuntimeOpts {
memory_limit_mb: 512,
worker_timeout_ms: 5 * 60 * 1000,
low_memory_multiplier: 5,
max_cpu_bursts: 10,
cpu_burst_interval_ms: 100,
cpu_time_threshold_ms: 50,

force_create: false,
key: None,
pool_msg_tx: None,
events_msg_tx: None,
}
}
}

#[derive(Debug)]
pub enum UserWorkerMsgs {
Create(
Expand Down
6 changes: 6 additions & 0 deletions crates/sb_workers/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ pub struct UserWorkerCreateOptions {
import_map_path: Option<String>,
env_vars: Vec<(String, String)>,
force_create: bool,
allow_remote_modules: bool,
custom_module_root: Option<String>,

memory_limit_mb: u64,
low_memory_multiplier: u64,
Expand All @@ -71,6 +73,8 @@ pub async fn op_user_worker_create(
import_map_path,
env_vars,
force_create,
allow_remote_modules,
custom_module_root,

memory_limit_mb,
low_memory_multiplier,
Expand Down Expand Up @@ -98,6 +102,8 @@ pub async fn op_user_worker_create(
max_cpu_bursts,
cpu_burst_interval_ms,
force_create,
allow_remote_modules,
custom_module_root,
key: None,
pool_msg_tx: None,
events_msg_tx: None,
Expand Down
133 changes: 66 additions & 67 deletions crates/sb_workers/user_workers.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
const primordials = globalThis.__bootstrap.primordials;
const {
TypeError
TypeError,
} = primordials;
import {
readableStreamForRid,
writableStreamForRid,
} from "ext:deno_web/06_streams.js";
import { readableStreamForRid, writableStreamForRid } from 'ext:deno_web/06_streams.js';
const core = globalThis.Deno.core;
const ops = core.ops;

Expand All @@ -21,72 +18,74 @@ const ops = core.ops;
const chunkExpression = /(?:^|\W)chunked(?:$|\W)/i;

function nullBodyStatus(status) {
return status === 101 || status === 204 || status === 205 || status === 304;
return status === 101 || status === 204 || status === 205 || status === 304;
}

class UserWorker {
constructor(key) {
this.key = key;
}

async fetch(req) {
const { method, url, headers, body, bodyUsed } = req;

const headersArray = Array.from(headers.entries());
const hasReqBody = !bodyUsed && !!body && (chunkExpression.test(headers.get("transfer-encoding")) ||
Number.parseInt(headers.get("content-length"), 10) > 0);

const userWorkerReq = {
method,
url,
headers: headersArray,
hasBody: hasReqBody,
};

const { requestRid, requestBodyRid } = await ops.op_user_worker_fetch_build(userWorkerReq);

// stream the request body
if (hasReqBody) {
let writableStream = writableStreamForRid(requestBodyRid);
body.pipeTo(writableStream);
}

const res = await core.opAsync("op_user_worker_fetch_send", this.key, requestRid);
const bodyStream = readableStreamForRid(res.bodyRid);
return new Response(nullBodyStatus(res.status) ? null : bodyStream, {
headers: res.headers,
status: res.status,
statusText: res.statusText
});
}

static async create(opts) {
const readyOptions = {
memoryLimitMb: 512,
lowMemoryMultiplier: 5,
workerTimeoutMs: 5 * 60 * 1000,
cpuTimeThresholdMs: 50,
cpuBurstIntervalMs: 100,
maxCpuBursts: 10,
noModuleCache: false,
importMapPath: null,
envVars: [],
forceCreate: false,
...opts
}

const { servicePath } = readyOptions;

if (!servicePath || servicePath === "") {
throw new TypeError("service path must be defined");
}

const key = await core.opAsync("op_user_worker_create", readyOptions);

return new UserWorker(key);
}
constructor(key) {
this.key = key;
}

async fetch(req) {
const { method, url, headers, body, bodyUsed } = req;

const headersArray = Array.from(headers.entries());
const hasReqBody = !bodyUsed && !!body &&
(chunkExpression.test(headers.get('transfer-encoding')) ||
Number.parseInt(headers.get('content-length'), 10) > 0);

const userWorkerReq = {
method,
url,
headers: headersArray,
hasBody: hasReqBody,
};

const { requestRid, requestBodyRid } = await ops.op_user_worker_fetch_build(userWorkerReq);

// stream the request body
if (hasReqBody) {
let writableStream = writableStreamForRid(requestBodyRid);
body.pipeTo(writableStream);
}

const res = await core.opAsync('op_user_worker_fetch_send', this.key, requestRid);
const bodyStream = readableStreamForRid(res.bodyRid);
return new Response(nullBodyStatus(res.status) ? null : bodyStream, {
headers: res.headers,
status: res.status,
statusText: res.statusText,
});
}

static async create(opts) {
const readyOptions = {
memoryLimitMb: 512,
lowMemoryMultiplier: 5,
workerTimeoutMs: 5 * 60 * 1000,
cpuTimeThresholdMs: 50,
cpuBurstIntervalMs: 100,
maxCpuBursts: 10,
noModuleCache: false,
importMapPath: null,
envVars: [],
forceCreate: false,
allowRemoteModules: true,
customModuleRoot: null,
...opts,
};

const { servicePath } = readyOptions;

if (!servicePath || servicePath === '') {
throw new TypeError('service path must be defined');
}

const key = await core.opAsync('op_user_worker_create', readyOptions);

return new UserWorker(key);
}
}

const SUPABASE_USER_WORKERS = UserWorker;
export { SUPABASE_USER_WORKERS };

2 changes: 2 additions & 0 deletions examples/main/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ serve(async (req: Request) => {
const memoryLimitMb = 150;
const workerTimeoutMs = 10 * 1000;
const noModuleCache = false;
const customModuleRoot = null;
// you can provide an import map inline
// const inlineImportMap = {
// imports: {
Expand All @@ -53,6 +54,7 @@ serve(async (req: Request) => {
importMapPath,
envVars,
forceCreate,
customModuleRoot,
});
};

Expand Down

0 comments on commit a03877b

Please sign in to comment.