Merge #2135

2135: Distro packaging improvements r=Hywan a=jcaesar

# Description
Having installed wasmer from both gentoo and AUR (packaged by svenstaro, an Arch Maintainer), I was stumped to find that both build wasmer without compilers, not actually allowing me to run any wasm. (The AUR PKGBUILD has since been patched, but the gentoo ebuild maintainer… who knows.)

I figured that the obvious improvement would be to deny compilation if not at least one compiler is enabled, but I think more can be done to help packaging. I added
* a way to set a default for `WASMER_DIR` at compile time
* an install target for the Makefile
* a `PACKAGING.md` with some suggestions to distro maintainers

Disclaimer: I'm not a package maintainer anywhere, so these might be entirely off the rails. Commit 1 and 2 seem fairly reasonable to me, but for the other two, I'm not so sure.

# Review

- [ ] Add a short description of the the change to the CHANGELOG.md file


Co-authored-by: Julius Michaelis <[email protected]>
Co-authored-by: Ivan Enderlin <[email protected]>

CHANGELOG.md

@@ -11,6 +11,7 @@
 - [#2123](https://github.com/wasmerio/wasmer/pull/2123) Use `ENABLE_{{compiler_name}}=(0|1)` to resp. force to disable or enable a compiler when running the `Makefile`, e.g. `ENABLE_LLVM=1 make build-wasmer`.
 - [#2123](https://github.com/wasmerio/wasmer/pull/2123) `libwasmer` comes with all available compilers per target instead of Cranelift only.
 - [#2118](https://github.com/wasmerio/wasmer/pull/2118) Add an unstable non-standard C API to query available engines and compilers.
+- [#2135](https://github.com/wasmerio/wasmer/pull/2135) [Documentation](./PACKAGING.md) for linux distribution maintainers
 
 ### Changed
 - [#2113](https://github.com/wasmerio/wasmer/pull/2113) Bump minimum supported Rust version to 1.49

Makefile

@@ -330,6 +330,9 @@ $(info )
 # Building #
 ############
 
+# Not really "all", just the default target that builds enough so make install will go through
+all: build-wasmer build-capi
+
 bench:
 	cargo bench $(compiler_features)
 
@@ -495,7 +498,7 @@ test-packages:
 	cargo test -p wasmer-engine-native --release --no-default-features
 	cargo test -p wasmer-engine-jit --release --no-default-features
 	cargo test -p wasmer-compiler --release
-	cargo test -p wasmer-cli --release
+	cargo test --manifest-path lib/cli/Cargo.toml $(compiler_features) --release
 	cargo test -p wasmer-cache --release
 	cargo test -p wasmer-engine --release
 	cargo test -p wasmer-derive --release
@@ -659,6 +662,44 @@ endif
 	tar -C package -zcvf wasmer.tar.gz bin lib include LICENSE ATTRIBUTIONS
 	mv wasmer.tar.gz dist/
 
+########################
+# (Distro-) Installing #
+########################
+
+DESTDIR ?= /usr/local
+
+install: install-wasmer install-capi-headers install-capi-lib install-capi-staticlib install-pkgconfig install-misc
+
+install-wasmer:
+	install -Dm755 target/release/wasmer $(DESTDIR)/bin/wasmer
+
+install-capi-headers:
+	for header in lib/c-api/*.{h,hh}; do install -Dm644 "$$header" $(DESTDIR)/include/$$(basename $$header); done
+	install -Dm644 lib/c-api/README.md $(DESTDIR)/include/wasmer-README.md
+
+# Currently implemented for linux only. TODO
+install-capi-lib:
+	pkgver=$$(target/release/wasmer --version | cut -d\  -f2) && \
+	shortver="$${pkgver%.*}" && \
+	majorver="$${shortver%.*}" && \
+	install -Dm755 target/release/libwasmer_c_api.so "$(DESTDIR)/lib/libwasmer.so.$$pkgver" && \
+	ln -sf "libwasmer.so.$$pkgver" "$(DESTDIR)/lib/libwasmer.so.$$shortver" && \
+	ln -sf "libwasmer.so.$$pkgver" "$(DESTDIR)/lib/libwasmer.so.$$majorver" && \
+	ln -sf "libwasmer.so.$$pkgver" "$(DESTDIR)/lib/libwasmer.so"
+
+install-capi-staticlib:
+	install -Dm644 target/release/libwasmer_c_api.a "$(DESTDIR)/lib/libwasmer.a"
+
+install-misc:
+	install -Dm644 LICENSE "$(DESTDIR)"/share/licenses/wasmer/LICENSE
+
+install-pkgconfig:
+	unset WASMER_DIR # Make sure WASMER_INSTALL_PREFIX is set during build
+	target/release/wasmer config --pkg-config | install -Dm644 /dev/stdin "$(DESTDIR)"/lib/pkgconfig/wasmer.pc
+
+install-wasmer-headless-minimal:
+	install -Dm755 target/release/wasmer-headless $(DESTDIR)/bin/wasmer-headless
+
 #################
 # Miscellaneous #
 #################
@@ -680,7 +721,7 @@ lint-packages:
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-compiler
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-compiler-cranelift
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-compiler-singlepass
-	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-cli
+	RUSTFLAGS=${RUSTFLAGS} cargo clippy --manifest-path lib/cli/Cargo.toml $(compiler_features)
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-cache
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-engine
 

PACKAGING.md

@@ -0,0 +1,18 @@
+## Wasmer distro packaging notes
+
+* Where possible, do not directly invoke cargo, but use the supplied Makefile
+	* wasmer has several compiler backends and the Makefile autodetects whether to enable llvm and singlepass.
+	  Set `ENABLE_{CRANELIFT,LLVM,SINGLEPASS}=1` to build the full set or fail trying
+	* Set `WASMER_CAPI_USE_SYSTEM_LIBFFI=1` to force dynamic linking of libffi on the shared library
+	* `make install` respects `DESTDIR`, but `prefix` must be configured as e.g. `WASMER_INSTALL_PREFIX=/usr make all`
+* In case you must build/install directly with cargo, make sure to enable at least one compiler backend feature
+  * Beware that compiling with `cargo build --workspace/--all --features ...` will not enable features on the subcrates in the workspace and result in a headless wasmer binary that can not run wasm files directly.
+* If you split the package into several subpackages, beware that the create-exe command of wasmer requires `libwasmer.a` to be installed at `$WASMER_INSTALL_PREFIX/lib/libwasmer.a`.
+  Suggestion for splitting:
+  * `wasmer` and `wasmer-headless`, containing the respective executables
+    * `wasmer-headless` contains a subset of `wasmer`'s functionality and should only be packaged when splitting - it must be built explicitly with `make build-wasmer-headless-minimal install-wasmer-headless-minimal`
+  * `libwasmer`, containing `libwasmer.so*`
+  * `libwasmer-dev`, containing the header files and a `.pc` file
+  * `libwasmer-static`, containing `libwasmer.a`
+
+The wasmer distro packaging story is still in its infancy, so feedback is very welcome.

lib/cli/Cargo.toml

@@ -10,6 +10,7 @@ license = "MIT"
 readme = "README.md"
 edition = "2018"
 default-run = "wasmer"
+build = "build.rs"
 
 [[bin]]
 name = "wasmer"

lib/cli/build.rs

@@ -0,0 +1,4 @@
+pub fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=WASMER_INSTALL_PREFIX");
+}

lib/cli/src/bin/wasmer.rs

@@ -1,5 +1,10 @@
 use wasmer_cli::cli::wasmer_main;
 
+#[cfg(not(any(feature = "cranelift", feature = "singlepass", feature = "llvm")))]
+compile_error!(
+    "Either enable at least one compiler, or compile the wasmer-headless binary instead"
+);
+
 fn main() {
     wasmer_main();
 }

lib/cli/src/commands/config.rs

@@ -45,10 +45,16 @@ impl Config {
     }
     fn inner_execute(&self) -> Result<()> {
         let key = "WASMER_DIR";
-        let wasmer_dir = env::var(key).context(format!(
-            "failed to retrieve the {} environment variables",
-            key
-        ))?;
+        let wasmer_dir = env::var(key)
+            .or_else(|e| {
+                option_env!("WASMER_INSTALL_PREFIX")
+                    .map(str::to_string)
+                    .ok_or(e)
+            })
+            .context(format!(
+                "failed to retrieve the {} environment variables",
+                key
+            ))?;
 
         let prefix = PathBuf::from(wasmer_dir);
 

lib/cli/src/commands/create_exe.rs

@@ -148,14 +148,20 @@ fn generate_header(header_file_src: &[u8]) -> anyhow::Result<()> {
         .open(&header_file_path)?;
 
     use std::io::Write;
-    header.write(header_file_src)?;
+    header.write_all(header_file_src)?;
 
     Ok(())
 }
 
 fn get_wasmer_dir() -> anyhow::Result<PathBuf> {
     Ok(PathBuf::from(
-        env::var("WASMER_DIR").context("Trying to read env var `WASMER_DIR`")?,
+        env::var("WASMER_DIR")
+            .or_else(|e| {
+                option_env!("WASMER_INSTALL_PREFIX")
+                    .map(str::to_string)
+                    .ok_or(e)
+            })
+            .context("Trying to read env var `WASMER_DIR`")?,
     ))
 }