From 34e3164261fb39b4b788c7cdd0146b92fd73d85d Mon Sep 17 00:00:00 2001 From: "malonso@cloudflare.com" Date: Wed, 16 Dec 2020 16:47:42 -0600 Subject: [PATCH] support uploading module-based scripts --- src/upload/form/mod.rs | 100 ++++++++++++++++++++++++------ src/upload/form/modules_worker.rs | 75 ++++++++++++++++++++++ src/upload/form/project_assets.rs | 95 +++++++++++++++++++++++++++- src/upload/form/text_blob.rs | 3 + src/upload/form/wasm_module.rs | 7 ++- src/upload/package.rs | 22 ++++++- 6 files changed, 275 insertions(+), 27 deletions(-) create mode 100644 src/upload/form/modules_worker.rs diff --git a/src/upload/form/mod.rs b/src/upload/form/mod.rs index dc2ab2bc5..ddc1b964d 100644 --- a/src/upload/form/mod.rs +++ b/src/upload/form/mod.rs @@ -1,3 +1,4 @@ +mod modules_worker; mod plain_text; mod project_assets; mod service_worker; @@ -9,19 +10,23 @@ use std::fs; use std::path::Path; use std::path::PathBuf; +use ignore::WalkBuilder; + use crate::settings::binding; -use crate::settings::toml::{Target, TargetType}; +use crate::settings::toml::{ScriptFormat, Target, TargetType}; use crate::sites::AssetManifest; use crate::wranglerjs; use plain_text::PlainText; -use project_assets::ServiceWorkerAssets; +use project_assets::{ModulesAssets, ServiceWorkerAssets}; use text_blob::TextBlob; use wasm_module::WasmModule; // TODO: https://github.com/cloudflare/wrangler/issues/1083 use super::{krate, Package}; +use self::project_assets::Module; + pub fn build( target: &Target, asset_manifest: Option, @@ -63,23 +68,73 @@ pub fn build( service_worker::build_form(&assets, session_config) } - TargetType::JavaScript => { - log::info!("JavaScript project detected. Publishing..."); - let package_dir = target.package_dir()?; - let package = Package::new(&package_dir)?; - - let script_path = package.main(&package_dir)?; - - let assets = ServiceWorkerAssets::new( - script_path, - wasm_modules, - kv_namespaces.to_vec(), - text_blobs, - plain_texts, - )?; - - service_worker::build_form(&assets, session_config) - } + TargetType::JavaScript => match &target.builder_config { + Some(config) => match &config.upload_format { + ScriptFormat::ServiceWorker => { + log::info!("Plain JavaScript project detected. Publishing..."); + let package_dir = target.package_dir()?; + let package = Package::new(&package_dir)?; + let script_path = package.main(&package_dir)?; + + let assets = ServiceWorkerAssets::new( + script_path, + wasm_modules, + kv_namespaces.to_vec(), + text_blobs, + plain_texts, + )?; + + service_worker::build_form(&assets, session_config) + } + ScriptFormat::Modules => { + let package_dir = target.package_dir()?; + let package = Package::new(&package_dir)?; + let main_module = package.module(&package_dir)?; + let main_module_name = filename_from_path(&main_module) + .ok_or_else(|| failure::err_msg("filename required for main module"))?; + + let modules_iter = WalkBuilder::new(config.build_dir.clone()) + .standard_filters(false) + .build(); + + let mut modules: Vec = vec![]; + + for entry in modules_iter { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + modules.push(Module::new(path.to_owned())?); + } + } + + let assets = ModulesAssets::new( + main_module_name, + modules, + kv_namespaces.to_vec(), + plain_texts, + )?; + + modules_worker::build_form(&assets, session_config) + } + }, + None => { + log::info!("Plain JavaScript project detected. Publishing..."); + let package_dir = target.package_dir()?; + let package = Package::new(&package_dir)?; + + let script_path = package.main(&package_dir)?; + + let assets = ServiceWorkerAssets::new( + script_path, + wasm_modules, + kv_namespaces.to_vec(), + text_blobs, + plain_texts, + )?; + + service_worker::build_form(&assets, session_config) + } + }, TargetType::Webpack => { log::info!("webpack project detected. Publishing..."); // TODO: https://github.com/cloudflare/wrangler/issues/850 @@ -121,10 +176,15 @@ fn get_asset_manifest_blob(asset_manifest: AssetManifest) -> Result Option { +fn filestem_from_path(path: &PathBuf) -> Option { path.file_stem()?.to_str().map(|s| s.to_string()) } +fn filename_from_path(path: &PathBuf) -> Option { + path.file_name() + .map(|filename| filename.to_string_lossy().into_owned()) +} + fn build_generated_dir() -> Result<(), failure::Error> { let dir = "./worker/generated"; if !Path::new(dir).is_dir() { diff --git a/src/upload/form/modules_worker.rs b/src/upload/form/modules_worker.rs new file mode 100644 index 000000000..aa4abb5d4 --- /dev/null +++ b/src/upload/form/modules_worker.rs @@ -0,0 +1,75 @@ +use std::fs::File; + +use reqwest::blocking::multipart::{Form, Part}; +use serde::Serialize; + +use crate::settings::binding::Binding; + +use super::ModulesAssets; + +#[derive(Serialize, Debug)] +struct Metadata { + pub main_module: String, + pub bindings: Vec, +} + +pub fn build_form( + assets: &ModulesAssets, + session_config: Option, +) -> Result { + let mut form = Form::new(); + + // The preview service in particular streams the request form, and requires that the + // "metadata" part be set first, so this order is important. + form = add_metadata(form, assets)?; + form = add_files(form, assets)?; + if let Some(session_config) = session_config { + form = add_session_config(form, session_config)? + } + + log::info!("building form"); + log::info!("{:?}", &form); + + Ok(form) +} + +fn add_files(mut form: Form, assets: &ModulesAssets) -> Result { + for module in &assets.modules { + let file_name = module + .filename() + .ok_or_else(|| failure::err_msg("a filename is required for each module"))?; + let part = Part::reader(File::open(module.path.clone())?) + .mime_str(module.module_type.content_type())? + .file_name(file_name.clone()); + form = form.part(file_name.clone(), part); + } + Ok(form) +} + +fn add_metadata(mut form: Form, assets: &ModulesAssets) -> Result { + let metadata_json = serde_json::json!(&Metadata { + main_module: assets.main_module.clone(), + bindings: assets.bindings(), + }); + + let metadata = Part::text((metadata_json).to_string()) + .file_name("metadata.json") + .mime_str("application/json")?; + + form = form.part("metadata", metadata); + + Ok(form) +} + +fn add_session_config( + mut form: Form, + session_config: serde_json::Value, +) -> Result { + let wrangler_session_config = Part::text((session_config).to_string()) + .file_name("") + .mime_str("application/json")?; + + form = form.part("wrangler-session-config", wrangler_session_config); + + Ok(form) +} diff --git a/src/upload/form/project_assets.rs b/src/upload/form/project_assets.rs index 9ffdab502..fceed276d 100644 --- a/src/upload/form/project_assets.rs +++ b/src/upload/form/project_assets.rs @@ -3,10 +3,10 @@ use std::path::PathBuf; use failure::format_err; use super::binding::Binding; -use super::filename_from_path; use super::plain_text::PlainText; use super::text_blob::TextBlob; use super::wasm_module::WasmModule; +use super::{filename_from_path, filestem_from_path}; use crate::settings::toml::KvNamespace; @@ -28,7 +28,7 @@ impl ServiceWorkerAssets { text_blobs: Vec, plain_texts: Vec, ) -> Result<Self, failure::Error> { - let script_name = filename_from_path(&script_path).ok_or_else(|| { + let script_name = filestem_from_path(&script_path).ok_or_else(|| { format_err!("filename should not be empty: {}", script_path.display()) })?; @@ -73,3 +73,94 @@ impl ServiceWorkerAssets { self.script_path.clone() } } + +pub struct Module { + pub path: PathBuf, + pub module_type: ModuleType, +} + +impl Module { + pub fn new(path: PathBuf) -> Result<Module, failure::Error> { + let extension = path + .extension() + .ok_or(failure::err_msg( + "extension required to determine module type", + ))? + .to_string_lossy(); + + let module_type = match extension.as_ref() { + "mjs" => ModuleType::ES6, + "js" => ModuleType::CommonJS, + "wasm" => ModuleType::Wasm, + "txt" => ModuleType::Text, + _ => ModuleType::Data, + }; + + Ok(Module { path, module_type }) + } + + pub fn filename(&self) -> Option<String> { + filename_from_path(&self.path) + } +} + +pub enum ModuleType { + ES6, + CommonJS, + Wasm, + Text, + Data, +} + +impl ModuleType { + pub fn content_type(&self) -> &str { + match &self { + Self::ES6 => "application/javascript+module", + Self::CommonJS => "application/javascript", + Self::Wasm => "application/wasm", + Self::Text => "text/plain", + Self::Data => "application/octet-stream", + } + } +} + +pub struct ModulesAssets { + pub main_module: String, + pub modules: Vec<Module>, + pub kv_namespaces: Vec<KvNamespace>, + pub plain_texts: Vec<PlainText>, +} + +impl ModulesAssets { + pub fn new( + main_module: String, + modules: Vec<Module>, + kv_namespaces: Vec<KvNamespace>, + plain_texts: Vec<PlainText>, + ) -> Result<Self, failure::Error> { + Ok(Self { + main_module, + modules, + kv_namespaces, + plain_texts, + }) + } + + pub fn bindings(&self) -> Vec<Binding> { + let mut bindings = Vec::new(); + + // Bindings that refer to a `part` of the uploaded files + // in the service-worker format, are now modules. + + for kv in &self.kv_namespaces { + let binding = kv.binding(); + bindings.push(binding); + } + for plain_text in &self.plain_texts { + let binding = plain_text.binding(); + bindings.push(binding); + } + + bindings + } +} diff --git a/src/upload/form/text_blob.rs b/src/upload/form/text_blob.rs index e6c7e4694..921c8a820 100644 --- a/src/upload/form/text_blob.rs +++ b/src/upload/form/text_blob.rs @@ -1,6 +1,9 @@ use super::binding::Binding; use serde::{Deserialize, Serialize}; +// Note: This is only used for service-worker scripts. +// modules scripts use the universal Module class instead of this. + #[derive(Debug, Deserialize, Serialize)] pub struct TextBlob { pub data: String, diff --git a/src/upload/form/wasm_module.rs b/src/upload/form/wasm_module.rs index bb1406ddf..e5b93b95a 100644 --- a/src/upload/form/wasm_module.rs +++ b/src/upload/form/wasm_module.rs @@ -3,7 +3,10 @@ use std::path::PathBuf; use failure::format_err; use super::binding::Binding; -use super::filename_from_path; +use super::filestem_from_path; + +// Note: This is only used for service-worker scripts. +// modules scripts use the universal Module class instead of this. #[derive(Debug)] pub struct WasmModule { @@ -14,7 +17,7 @@ pub struct WasmModule { impl WasmModule { pub fn new(path: PathBuf, binding: String) -> Result<Self, failure::Error> { - let filename = filename_from_path(&path) + let filename = filestem_from_path(&path) .ok_or_else(|| format_err!("filename should not be empty: {}", path.display()))?; Ok(Self { diff --git a/src/upload/package.rs b/src/upload/package.rs index 33b293a45..f48ca79cb 100644 --- a/src/upload/package.rs +++ b/src/upload/package.rs @@ -7,6 +7,8 @@ use serde::{self, Deserialize}; pub struct Package { #[serde(default)] main: PathBuf, + #[serde(default)] + module: PathBuf, } impl Package { pub fn main(&self, package_dir: &PathBuf) -> Result<PathBuf, failure::Error> { @@ -23,16 +25,30 @@ impl Package { Ok(self.main.clone()) } } + pub fn module(&self, package_dir: &PathBuf) -> Result<PathBuf, failure::Error> { + if self.module == PathBuf::from("") { + failure::bail!( + "The `module` key in your `package.json` file is required when using the module script format; please specify the entry point of your Worker.", + ) + } else if !package_dir.join(&self.module).exists() { + failure::bail!( + "The entrypoint of your Worker ({}) could not be found.", + self.module.display() + ) + } else { + Ok(self.module.clone()) + } + } } impl Package { - pub fn new(pkg_path: &PathBuf) -> Result<Package, failure::Error> { - let manifest_path = pkg_path.join("package.json"); + pub fn new(package_dir: &PathBuf) -> Result<Package, failure::Error> { + let manifest_path = package_dir.join("package.json"); if !manifest_path.is_file() { failure::bail!( "Your JavaScript project is missing a `package.json` file; is `{}` the \ wrong directory?", - pkg_path.display() + package_dir.display() ) }