From dcaa2c1cffaa24b06f0d38a009bb0020bd4816f2 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Tue, 31 Oct 2023 17:30:13 +0100 Subject: [PATCH] feat(cli): Add package build/download/unpack commands Adds a new CLI command for working with packages: * package download: download a package from a registry * build: build a package from a wasmer.toml manifest * unpack: extract package contents --- Cargo.lock | 6 +- Cargo.toml | 2 +- lib/cli/Cargo.toml | 60 ++------ lib/cli/src/cli.rs | 13 +- lib/cli/src/commands.rs | 3 +- lib/cli/src/commands/package/build.rs | 131 +++++++++++++++++ lib/cli/src/commands/package/download.rs | 173 +++++++++++++++++++++++ lib/cli/src/commands/package/mod.rs | 16 +++ lib/cli/src/commands/package/unpack.rs | 90 ++++++++++++ lib/registry/src/queries.rs | 0 lib/registry/src/wasmer_env.rs | 24 ++++ 11 files changed, 462 insertions(+), 56 deletions(-) create mode 100644 lib/cli/src/commands/package/build.rs create mode 100644 lib/cli/src/commands/package/download.rs create mode 100644 lib/cli/src/commands/package/mod.rs create mode 100644 lib/cli/src/commands/package/unpack.rs delete mode 100644 lib/registry/src/queries.rs diff --git a/Cargo.lock b/Cargo.lock index 9c567a71b40..8b358e6f3ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5680,7 +5680,9 @@ dependencies = [ "dirs", "distance", "flate2", + "futures", "hex", + "http", "hyper", "indexmap 1.9.3", "indicatif", @@ -6576,9 +6578,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.7.0" +version = "5.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5fc974157e8532c5c31fa44c38046264227e580cd2a0d939543891d23f43779" +checksum = "ac815d472f09ed064ef70a3046843972646727cbe55db795dd8a9925b76ed0c4" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index ce8c16815c3..5df335e4fab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,7 +84,7 @@ rust-version = "1.70" version = "4.2.3" [workspace.dependencies] -webc = { version = "5.5.1", default-features = false, features = ["package"] } +webc = { version = "5.8.0", default-features = false, features = ["package"] } wasmer-toml = "0.8.0" [build-dependencies] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index ead0d793f85..f000e756315 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -113,6 +113,8 @@ once_cell = "1.17.1" indicatif = "0.17.5" opener = "0.6.1" hyper = { version = "0.14.27", features = ["server"] } +http = "0.2.9" +futures = "0.3.29" # NOTE: Must use different features for clap because the "color" feature does not # work on wasi due to the anstream dependency not compiling. @@ -155,14 +157,7 @@ unix_mode = "0.1.3" [features] # Don't add the compiler features in default, please add them on the Makefile # since we might want to autoconfigure them depending on the availability on the host. -default = [ - "sys", - "wat", - "wast", - "compiler", - "wasmer-artifact-create", - "static-artifact-create", -] +default = ["sys", "wat", "wast", "compiler", "wasmer-artifact-create", "static-artifact-create"] backend = [] coredump = ["wasm-coredump-builder"] sys = ["compiler", "wasmer-vm"] @@ -170,56 +165,21 @@ jsc = ["backend", "wasmer/jsc", "wasmer/std"] wast = ["wasmer-wast"] host-net = ["virtual-net/host-net"] wat = ["wasmer/wat"] -compiler = [ - "backend", - "wasmer/compiler", - "wasmer-compiler/translator", - "wasmer-compiler/compiler", -] -wasmer-artifact-create = [ - "compiler", - "wasmer/wasmer-artifact-load", - "wasmer/wasmer-artifact-create", - "wasmer-compiler/wasmer-artifact-load", - "wasmer-compiler/wasmer-artifact-create", - "wasmer-object", -] -static-artifact-create = [ - "compiler", - "wasmer/static-artifact-load", - "wasmer/static-artifact-create", - "wasmer-compiler/static-artifact-load", - "wasmer-compiler/static-artifact-create", - "wasmer-object", -] -wasmer-artifact-load = [ - "compiler", - "wasmer/wasmer-artifact-load", - "wasmer-compiler/wasmer-artifact-load", -] -static-artifact-load = [ - "compiler", - "wasmer/static-artifact-load", - "wasmer-compiler/static-artifact-load", -] +compiler = ["backend", "wasmer/compiler", "wasmer-compiler/translator", "wasmer-compiler/compiler"] +wasmer-artifact-create = ["compiler", "wasmer/wasmer-artifact-load", "wasmer/wasmer-artifact-create", "wasmer-compiler/wasmer-artifact-load", "wasmer-compiler/wasmer-artifact-create", "wasmer-object"] +static-artifact-create = ["compiler", "wasmer/static-artifact-load", "wasmer/static-artifact-create", "wasmer-compiler/static-artifact-load", "wasmer-compiler/static-artifact-create", "wasmer-object"] +wasmer-artifact-load = ["compiler", "wasmer/wasmer-artifact-load", "wasmer-compiler/wasmer-artifact-load"] +static-artifact-load = ["compiler", "wasmer/static-artifact-load", "wasmer-compiler/static-artifact-load"] experimental-io-devices = ["wasmer-wasix-experimental-io-devices"] singlepass = ["wasmer-compiler-singlepass", "compiler"] cranelift = ["wasmer-compiler-cranelift", "compiler"] llvm = ["wasmer-compiler-llvm", "compiler"] -disable-all-logging = [ - "wasmer-wasix/disable-all-logging", - "log/release_max_level_off", -] +disable-all-logging = ["wasmer-wasix/disable-all-logging", "log/release_max_level_off"] headless = [] headless-minimal = ["headless", "disable-all-logging"] # Optional -enable-serde = [ - "wasmer/enable-serde", - "wasmer-vm/enable-serde", - "wasmer-compiler/enable-serde", - "wasmer-wasix/enable-serde", -] +enable-serde = ["wasmer/enable-serde", "wasmer-vm/enable-serde", "wasmer-compiler/enable-serde", "wasmer-wasix/enable-serde"] [dev-dependencies] assert_cmd = "2.0.11" diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 8b343f4587c..1575ee140dd 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -9,7 +9,8 @@ use crate::commands::CreateExe; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{ - Add, Cache, Config, Init, Inspect, Login, Publish, Run, SelfUpdate, Validate, Whoami, + Add, Cache, CmdPackage, Config, Init, Inspect, Login, Publish, Run, SelfUpdate, Validate, + Whoami, }; #[cfg(feature = "static-artifact-create")] use crate::commands::{CreateObj, GenCHeader}; @@ -108,6 +109,11 @@ impl Args { Some(Cmd::Init(init)) => init.execute(), Some(Cmd::Login(login)) => login.execute(), Some(Cmd::Publish(publish)) => publish.execute(), + Some(Cmd::Package(cmd)) => match cmd { + CmdPackage::Download(cmd) => cmd.execute(), + CmdPackage::Unpack(cmd) => cmd.execute(), + CmdPackage::Build(cmd) => cmd.execute(), + }, /* Some(Cmd::Connect(connect)) => connect.execute(), */ @@ -258,7 +264,10 @@ enum Cmd { #[clap(alias = "run-unstable")] Run(Run), - // DEPLOY commands + #[clap(subcommand)] + Package(crate::commands::CmdPackage), + + // Edge commands /// Deploy apps to the Wasmer Edge. Deploy(wasmer_deploy_cli::cmd::deploy::CmdDeploy), diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 9f2365837cd..11b632e5460 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -15,6 +15,7 @@ mod gen_c_header; mod init; mod inspect; mod login; +mod package; mod publish; mod run; mod self_update; @@ -32,7 +33,7 @@ pub use create_exe::*; #[cfg(feature = "wast")] pub use wast::*; pub use { - add::*, cache::*, config::*, init::*, inspect::*, login::*, publish::*, run::Run, + add::*, cache::*, config::*, init::*, inspect::*, login::*, package::*, publish::*, run::Run, self_update::*, validate::*, whoami::*, }; #[cfg(feature = "static-artifact-create")] diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs new file mode 100644 index 00000000000..f3ceade61ce --- /dev/null +++ b/lib/cli/src/commands/package/build.rs @@ -0,0 +1,131 @@ +use std::path::PathBuf; + +use anyhow::Context; + +/// Extract contents from a package file to a directory. +#[derive(clap::Parser, Debug)] +pub struct CmdPackageBuild { + /// Path of the package or wasmer.toml manifest. + /// + /// Defaults to current directory. + #[clap(short = 'p', long)] + package: Option, + + /// Output path for the package file. + /// Defaults to current directory + [name]-[version].webc. + #[clap(short = 'o', long)] + out: Option, +} + +impl CmdPackageBuild { + pub(crate) fn execute(&self) -> Result<(), anyhow::Error> { + let manifest_path = if let Some(p) = &self.package { + if p.is_dir() { + let manifest_path = p.join("wasmer.toml"); + if !manifest_path.is_file() { + anyhow::bail!( + "Specified directory '{}' does not contain a wasmer.toml manifest", + p.display() + ); + } + manifest_path + } else if p.is_file() { + p.clone() + } else { + anyhow::bail!( + "Specified path '{}' is not a file or directory", + p.display() + ); + } + } else { + let dir = std::env::current_dir().context("could not get current directory")?; + let manifest_path = dir.join("wasmer.toml"); + if !manifest_path.is_file() { + anyhow::bail!( + "Current directory '{}' does not contain a wasmer.toml manifest - specify a path with --package-dir", + dir.display() + ); + } + manifest_path + }; + + let pkg = webc::wasmer_package::Package::from_manifest(&manifest_path)?; + + let manifest = pkg + .manifest() + .package_annotation::("wapm") + .context("could not load package metadata")? + .context("could not find package metadata")?; + + let pkgname = manifest.name.replace("/", "-"); + let name = format!("{}-{}.webc", pkgname, manifest.version,); + let out_path = if let Some(p) = &self.out { + if p.is_dir() { + p.join(name) + } else { + if let Some(parent) = p.parent() { + std::fs::create_dir_all(parent).context("could not create output directory")?; + } + + p.to_owned() + } + } else { + std::env::current_dir() + .context("could not determine current directory")? + .join(name) + }; + + if out_path.exists() { + anyhow::bail!( + "Output path '{}' already exists - specify a different path with -o/--out", + out_path.display() + ); + } + + let data = pkg.serialize()?; + std::fs::write(&out_path, &data) + .with_context(|| format!("could not write contents to '{}'", out_path.display()))?; + + eprintln!("Package written to '{}'", out_path.display()); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Download a package from the dev registry. + #[test] + fn test_cmd_package_build() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path(); + + std::fs::write( + path.join("wasmer.toml"), + r#" +[package] +name = "wasmer/hello" +version = "0.1.0" +description = "hello" + +[fs] +"data" = "data" +"#, + ) + .unwrap(); + + std::fs::create_dir(path.join("data")).unwrap(); + std::fs::write(path.join("data").join("hello.txt"), "Hello, world!").unwrap(); + + let cmd = CmdPackageBuild { + package: Some(path.to_owned()), + out: Some(path.to_owned()), + }; + + cmd.execute().unwrap(); + + webc::Container::from_disk(path.join("wasmer-hello-0.1.0.webc")).unwrap(); + } +} diff --git a/lib/cli/src/commands/package/download.rs b/lib/cli/src/commands/package/download.rs new file mode 100644 index 00000000000..94eef3ad2c1 --- /dev/null +++ b/lib/cli/src/commands/package/download.rs @@ -0,0 +1,173 @@ +use std::path::PathBuf; + +use anyhow::Context; +use futures::StreamExt; +use tokio::io::AsyncWriteExt; +use wasmer_registry::wasmer_env::WasmerEnv; +use wasmer_wasix::runtime::resolver::PackageSpecifier; + +/// Download a package from the registry. +#[derive(clap::Parser, Debug)] +pub struct CmdPackageDownload { + #[clap(flatten)] + env: WasmerEnv, + + /// Verify that the downloaded file is a valid package. + #[clap(long)] + validate: bool, + + /// Path where the package file should be written to. + /// If not specified, the data will be written to stdout. + #[clap(short = 'o', long)] + out_path: PathBuf, + + /// The package to download. + /// Can be: + /// * a pakage specifier: `namespace/package[@vesion]` + /// * a URL + package: PackageSpecifier, +} + +impl CmdPackageDownload { + pub(crate) fn execute(&self) -> Result<(), anyhow::Error> { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(self.run()) + } + + async fn run(&self) -> Result<(), anyhow::Error> { + if let Some(parent) = self.out_path.parent() { + match parent.metadata() { + Ok(m) => { + if !m.is_dir() { + anyhow::bail!( + "parent of output file is not a directory: '{}'", + parent.display() + ); + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir_all(parent) + .context("could not create parent directory of output file")?; + } + Err(err) => return Err(err.into()), + } + }; + + let (url, token) = match &self.package { + PackageSpecifier::Registry { full_name, version } => { + let mut url = self.env.registry_public_url()?; + let p = format!("/{full_name}@{version}"); + url.set_path(&p); + (url, self.env.token()) + } + PackageSpecifier::Url(url) => { + (url.clone(), self.env.get_token_opt().map(|x| x.to_string())) + } + PackageSpecifier::Path(_) => { + anyhow::bail!("cannot download a package from a local path"); + } + }; + + let client = reqwest::Client::new(); + let mut b = client + .get(url.clone()) + .header(http::header::ACCEPT, "application/webc"); + if let Some(token) = token { + b = b.header(http::header::AUTHORIZATION, format!("Bearer {token}")); + }; + + eprintln!("Downloading package from '{url}'..."); + + let res = b + .send() + .await + .context("http request failed")? + .error_for_status() + .context("http request failed with non-success status code")?; + + let time = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(); + + let tmp_name = url + .as_str() + .chars() + .map(|x| match x { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' => x, + _ => '_', + }) + .collect::(); + let tmp_path = std::env::temp_dir().join(format!("wasmer-download-{}-{}", tmp_name, time)); + let mut file = tokio::fs::File::create(&tmp_path).await.with_context(|| { + format!( + "could not create temporary file at '{}'", + tmp_path.display() + ) + })?; + + let ty = res + .headers() + .get(http::header::CONTENT_TYPE) + .and_then(|t| t.to_str().ok()) + .unwrap_or_default(); + + if !(ty == "application/webc" || ty == "application/octet-stream") { + eprintln!( + "Warning: response has invalid content type - expected \ + 'application/webc' or 'application/octet-stream', got {ty}" + ); + } + + let mut body = res.bytes_stream(); + + while let Some(res) = body.next().await { + let chunk = res.context("could not read response body")?; + // Yes, we are mixing async and sync code here, but since this is + // a top-level command, this can't interfere with other tasks. + file.write_all(&chunk) + .await + .context("could not write to temporary file")?; + } + + file.sync_all() + .await + .context("could not sync temporary file")?; + std::mem::drop(file); + + if self.validate { + eprintln!("Validating package..."); + webc::compat::Container::from_disk(&tmp_path) + .context("could not parse downloaded file as a package - invalid download?")?; + eprintln!("Downloaded package is valid!"); + } + + std::fs::rename(&tmp_path, &self.out_path) + .context("could not move temporary file to output path")?; + + eprintln!("Package downloaded to '{}'", self.out_path.display()); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use wasmer_registry::wasmer_env::WASMER_DIR; + + use super::*; + + /// Download a package from the dev registry. + #[test] + fn test_cmd_package_download() { + let dir = tempfile::tempdir().unwrap(); + + let cmd = CmdPackageDownload { + env: WasmerEnv::new(WASMER_DIR.clone(), Some("wasmer.wtf".into()), None, None), + validate: true, + out_path: dir.path().join("hello.webc"), + package: "wasmer/hello@0.1.0".parse().unwrap(), + }; + + cmd.execute().unwrap(); + } +} diff --git a/lib/cli/src/commands/package/mod.rs b/lib/cli/src/commands/package/mod.rs new file mode 100644 index 00000000000..0e332fc471d --- /dev/null +++ b/lib/cli/src/commands/package/mod.rs @@ -0,0 +1,16 @@ +mod build; +mod download; +mod unpack; + +pub use {download::CmdPackageDownload, unpack::CmdPackageUnpack}; + +/// Package related commands. +#[derive(clap::Subcommand, Debug)] +// Allowing missing_docs because the comment would override the doc comment on +// the command struct. +#[allow(missing_docs)] +pub enum CmdPackage { + Download(CmdPackageDownload), + Unpack(unpack::CmdPackageUnpack), + Build(build::CmdPackageBuild), +} diff --git a/lib/cli/src/commands/package/unpack.rs b/lib/cli/src/commands/package/unpack.rs new file mode 100644 index 00000000000..5b3c7fd1d93 --- /dev/null +++ b/lib/cli/src/commands/package/unpack.rs @@ -0,0 +1,90 @@ +use std::path::PathBuf; + +use anyhow::Context; + +/// Extract contents from a package file to a directory. +#[derive(clap::Parser, Debug)] +pub struct CmdPackageUnpack { + /// The output directory. + #[clap(short = 'o', long)] + out_dir: PathBuf, + + /// Overwrite existing directories/files. + #[clap(long)] + overwrite: bool, + + /// Path to the package. + package_path: PathBuf, +} + +impl CmdPackageUnpack { + pub(crate) fn execute(&self) -> Result<(), anyhow::Error> { + eprintln!("Unpacking..."); + + let pkg = webc::compat::Container::from_disk(&self.package_path).with_context(|| { + format!( + "could not open package at '{}'", + self.package_path.display() + ) + })?; + + let outdir = &self.out_dir; + std::fs::create_dir_all(&outdir) + .with_context(|| format!("could not create output directory '{}'", outdir.display()))?; + + pkg.unpack(&outdir, self.overwrite) + .with_context(|| format!("could not extract package"))?; + + eprintln!("Extracted package contents to '{}'", self.out_dir.display()); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Download a package from the dev registry. + #[test] + fn test_cmd_package_extract() { + let dir = tempfile::tempdir().unwrap(); + + let package_path = std::env::var("CARGO_MANIFEST_DIR").map(PathBuf::from).unwrap() + .parent().unwrap() + .parent().unwrap() + .join("tests/integration/cli/tests/webc/hello-0.1.0-665d2ddc-80e6-4845-85d3-4587b1693bb7.webc"); + + assert!(package_path.is_file()); + + let cmd = CmdPackageUnpack { + out_dir: dir.path().to_owned(), + overwrite: false, + package_path, + }; + + cmd.execute().unwrap(); + + let mut items = std::fs::read_dir(dir.path()) + .unwrap() + .map(|x| { + x.unwrap() + .path() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string() + }) + .collect::>(); + items.sort(); + assert_eq!( + items, + vec![ + "atom".to_string(), + "manifest.json".to_string(), + "metadata".to_string(), + ] + ); + } +} diff --git a/lib/registry/src/queries.rs b/lib/registry/src/queries.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/registry/src/wasmer_env.rs b/lib/registry/src/wasmer_env.rs index 110d434fdee..5ba71834693 100644 --- a/lib/registry/src/wasmer_env.rs +++ b/lib/registry/src/wasmer_env.rs @@ -42,6 +42,22 @@ impl WasmerEnv { } } + pub fn registry_public_url(&self) -> Result { + let mut url = self.registry_endpoint()?; + url.set_path(""); + + let domain = url + .host_str() + .context("url has no host")? + .strip_prefix("registry.") + .context("could not derive registry public url")? + .to_string(); + url.set_host(Some(&domain)) + .context("could not derive registry public url")?; + + Ok(url) + } + /// Get the GraphQL endpoint used to query the registry. pub fn registry_endpoint(&self) -> Result { if let Some(registry) = &self.registry { @@ -79,6 +95,14 @@ impl WasmerEnv { } } + /// Retrieve the specified token. + /// + /// NOTE: In contrast to [`Self::token`], this will not fall back to loading + /// the token from the confg file. + pub fn get_token_opt(&self) -> Option<&str> { + self.token.as_deref() + } + /// The API token for the active registry. pub fn token(&self) -> Option { if let Some(token) = &self.token {