Merge pull request #2135 from jcaesar/distro

Distro packaging improvements

.github/workflows/lint.yaml

@@ -22,7 +22,18 @@ jobs:
           toolchain: 1.49
           override: true
           components: rustfmt, clippy
+      - name: Install LLVM (Linux)
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/linux-amd64.tar.gz -L -o /opt/llvm.tar.xz
+          mkdir -p /opt/llvm-11
+          tar xf /opt/llvm.tar.xz --strip-components=1 -C /opt/llvm-11
+          echo '/opt/llvm-11/bin' >> $GITHUB_PATH
+          echo 'LLVM_SYS_110_PREFIX=/opt/llvm-11' >> $GITHUB_ENV
       - run: make lint
+        env:
+          ENABLE_CRANELIFT: "1"
+          ENABLE_LLVM: "1"
+          ENABLE_SINGLEPASS: "1"
       - name: Assert no files have changed
         run: |
           git status

CHANGELOG.md

@@ -12,6 +12,7 @@
 - [#2123](https://github.com/wasmerio/wasmer/pull/2123) Use `ENABLE_{{compiler_name}}=(0|1)` to resp. force to disable or enable a compiler when running the `Makefile`, e.g. `ENABLE_LLVM=1 make build-wasmer`.
 - [#2123](https://github.com/wasmerio/wasmer/pull/2123) `libwasmer` comes with all available compilers per target instead of Cranelift only.
 - [#2118](https://github.com/wasmerio/wasmer/pull/2118) Add an unstable non-standard C API to query available engines and compilers.
+- [#2135](https://github.com/wasmerio/wasmer/pull/2135) [Documentation](./PACKAGING.md) for linux distribution maintainers
 
 ### Changed
 - [#2113](https://github.com/wasmerio/wasmer/pull/2113) Bump minimum supported Rust version to 1.49

Makefile

@@ -330,6 +330,9 @@ $(info )
 # Building #
 ############
 
+# Not really "all", just the default target that builds enough so make install will go through
+all: build-wasmer build-capi
+
 bench:
 	cargo bench $(compiler_features)
 
@@ -495,7 +498,7 @@ test-packages:
 	cargo test -p wasmer-engine-native --release --no-default-features
 	cargo test -p wasmer-engine-jit --release --no-default-features
 	cargo test -p wasmer-compiler --release
-	cargo test -p wasmer-cli --release
+	cargo test --manifest-path lib/cli/Cargo.toml $(compiler_features) --release
 	cargo test -p wasmer-cache --release
 	cargo test -p wasmer-engine --release
 	cargo test -p wasmer-derive --release
@@ -659,6 +662,44 @@ endif
 	tar -C package -zcvf wasmer.tar.gz bin lib include LICENSE ATTRIBUTIONS
 	mv wasmer.tar.gz dist/
 
+########################
+# (Distro-) Installing #
+########################
+
+DESTDIR ?= /usr/local
+
+install: install-wasmer install-capi-headers install-capi-lib install-capi-staticlib install-pkgconfig install-misc
+
+install-wasmer:
+	install -Dm755 target/release/wasmer $(DESTDIR)/bin/wasmer
+
+install-capi-headers:
+	for header in lib/c-api/*.{h,hh}; do install -Dm644 "$$header" $(DESTDIR)/include/$$(basename $$header); done
+	install -Dm644 lib/c-api/README.md $(DESTDIR)/include/wasmer-README.md
+
+# Currently implemented for linux only. TODO
+install-capi-lib:
+	pkgver=$$(target/release/wasmer --version | cut -d\  -f2) && \
+	shortver="$${pkgver%.*}" && \
+	majorver="$${shortver%.*}" && \
+	install -Dm755 target/release/libwasmer_c_api.so "$(DESTDIR)/lib/libwasmer.so.$$pkgver" && \
+	ln -sf "libwasmer.so.$$pkgver" "$(DESTDIR)/lib/libwasmer.so.$$shortver" && \
+	ln -sf "libwasmer.so.$$pkgver" "$(DESTDIR)/lib/libwasmer.so.$$majorver" && \
+	ln -sf "libwasmer.so.$$pkgver" "$(DESTDIR)/lib/libwasmer.so"
+
+install-capi-staticlib:
+	install -Dm644 target/release/libwasmer_c_api.a "$(DESTDIR)/lib/libwasmer.a"
+
+install-misc:
+	install -Dm644 LICENSE "$(DESTDIR)"/share/licenses/wasmer/LICENSE
+
+install-pkgconfig:
+	unset WASMER_DIR # Make sure WASMER_INSTALL_PREFIX is set during build
+	target/release/wasmer config --pkg-config | install -Dm644 /dev/stdin "$(DESTDIR)"/lib/pkgconfig/wasmer.pc
+
+install-wasmer-headless-minimal:
+	install -Dm755 target/release/wasmer-headless $(DESTDIR)/bin/wasmer-headless
+
 #################
 # Miscellaneous #
 #################
@@ -680,7 +721,7 @@ lint-packages:
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-compiler
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-compiler-cranelift
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-compiler-singlepass
-	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-cli
+	RUSTFLAGS=${RUSTFLAGS} cargo clippy --manifest-path lib/cli/Cargo.toml $(compiler_features)
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-cache
 	RUSTFLAGS=${RUSTFLAGS} cargo clippy -p wasmer-engine
 

PACKAGING.md

@@ -0,0 +1,18 @@
+## Wasmer distro packaging notes
+
+* Where possible, do not directly invoke cargo, but use the supplied Makefile
+	* wasmer has several compiler backends and the Makefile autodetects whether to enable llvm and singlepass.
+	  Set `ENABLE_{CRANELIFT,LLVM,SINGLEPASS}=1` to build the full set or fail trying
+	* Set `WASMER_CAPI_USE_SYSTEM_LIBFFI=1` to force dynamic linking of libffi on the shared library
+	* `make install` respects `DESTDIR`, but `prefix` must be configured as e.g. `WASMER_INSTALL_PREFIX=/usr make all`
+* In case you must build/install directly with cargo, make sure to enable at least one compiler backend feature
+  * Beware that compiling with `cargo build --workspace/--all --features ...` will not enable features on the subcrates in the workspace and result in a headless wasmer binary that can not run wasm files directly.
+* If you split the package into several subpackages, beware that the create-exe command of wasmer requires `libwasmer.a` to be installed at `$WASMER_INSTALL_PREFIX/lib/libwasmer.a`.
+  Suggestion for splitting:
+  * `wasmer` and `wasmer-headless`, containing the respective executables
+    * `wasmer-headless` contains a subset of `wasmer`'s functionality and should only be packaged when splitting - it must be built explicitly with `make build-wasmer-headless-minimal install-wasmer-headless-minimal`
+  * `libwasmer`, containing `libwasmer.so*`
+  * `libwasmer-dev`, containing the header files and a `.pc` file
+  * `libwasmer-static`, containing `libwasmer.a`
+
+The wasmer distro packaging story is still in its infancy, so feedback is very welcome.

lib/cli/Cargo.toml

@@ -10,6 +10,7 @@ license = "MIT"
 readme = "README.md"
 edition = "2018"
 default-run = "wasmer"
+build = "build.rs"
 
 [[bin]]
 name = "wasmer"

lib/cli/build.rs

@@ -0,0 +1,4 @@
+pub fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=WASMER_INSTALL_PREFIX");
+}

lib/cli/src/bin/wasmer.rs

@@ -1,5 +1,10 @@
 use wasmer_cli::cli::wasmer_main;
 
+#[cfg(not(any(feature = "cranelift", feature = "singlepass", feature = "llvm")))]
+compile_error!(
+    "Either enable at least one compiler, or compile the wasmer-headless binary instead"
+);
+
 fn main() {
     wasmer_main();
 }

lib/cli/src/commands/config.rs

@@ -45,10 +45,16 @@ impl Config {
     }
     fn inner_execute(&self) -> Result<()> {
         let key = "WASMER_DIR";
-        let wasmer_dir = env::var(key).context(format!(
-            "failed to retrieve the {} environment variables",
-            key
-        ))?;
+        let wasmer_dir = env::var(key)
+            .or_else(|e| {
+                option_env!("WASMER_INSTALL_PREFIX")
+                    .map(str::to_string)
+                    .ok_or(e)
+            })
+            .context(format!(
+                "failed to retrieve the {} environment variables",
+                key
+            ))?;
 
         let prefix = PathBuf::from(wasmer_dir);
 

lib/cli/src/commands/create_exe.rs

@@ -148,14 +148,20 @@ fn generate_header(header_file_src: &[u8]) -> anyhow::Result<()> {
         .open(&header_file_path)?;
 
     use std::io::Write;
-    header.write(header_file_src)?;
+    header.write_all(header_file_src)?;
 
     Ok(())
 }
 
 fn get_wasmer_dir() -> anyhow::Result<PathBuf> {
     Ok(PathBuf::from(
-        env::var("WASMER_DIR").context("Trying to read env var `WASMER_DIR`")?,
+        env::var("WASMER_DIR")
+            .or_else(|e| {
+                option_env!("WASMER_INSTALL_PREFIX")
+                    .map(str::to_string)
+                    .ok_or(e)
+            })
+            .context("Trying to read env var `WASMER_DIR`")?,
     ))
 }