From 38a10b4f9d0e80d4114d41e88265852967793559 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sat, 19 Jun 2021 21:16:36 +0200 Subject: [PATCH] rust: compile `alloc` from the in-tree sources Includes: - Cross-building a new sysroot with our custom `alloc` in order to support `alloc` tests. - Documentation generation for in-tree `alloc`. - Split `RUSTDOC` command in two for clarity, since now we have to also handle `sysroot` differently. - Proper handling of both silent and verbose mode for tests and `cargo`. - Deny missing docs globally `alloc`. - Allow broken intra docs for `alloc`. - Do not format `rust/alloc` nor `rust/test`, including handling the case where `srctree` is a prent folder. - `make clean` now cleans `rust/doc` and `rust/test`. - Move allocs out of the second expansion section since it is in-tree now. - Other small cleanups. I want to clean this up further and factor a few things, but let's move ahead. Signed-off-by: Miguel Ojeda --- Makefile | 29 +++++++---- rust/Makefile | 133 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 125 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 03fbf5bf49c017..f5b496948cb92b 100644 --- a/Makefile +++ b/Makefile @@ -461,9 +461,11 @@ STRIP = $(CROSS_COMPILE)strip endif RUSTC = rustc RUSTC_BOOTSTRAP = 1 +RUSTDOC = rustdoc RUSTFMT = rustfmt CLIPPY_DRIVER = clippy-driver BINDGEN = bindgen +CARGO = cargo PAHOLE = pahole RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids LEX = flex @@ -525,7 +527,7 @@ KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \ -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \ -Cforce-unwind-tables=n -Ccodegen-units=1 \ -Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0 \ - -D unsafe_op_in_unsafe_fn + -Dunsafe_op_in_unsafe_fn -Wmissing_docs KBUILD_AFLAGS_KERNEL := KBUILD_CFLAGS_KERNEL := KBUILD_RUSTCFLAGS_KERNEL := @@ -543,14 +545,13 @@ else RUSTC_OR_CLIPPY_QUIET := RUSTC RUSTC_OR_CLIPPY = $(RUSTC) endif -export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY ifdef RUST_LIB_SRC export RUST_LIB_SRC endif export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC -export RUSTC RUSTC_BOOTSTRAP BINDGEN +export RUSTC RUSTC_BOOTSTRAP RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD @@ -1598,7 +1599,7 @@ endif # CONFIG_MODULES # Directories & files removed with 'make clean' CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ - compile_commands.json .thinlto-cache + compile_commands.json .thinlto-cache rust/test rust/doc # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -1728,7 +1729,7 @@ help: @echo ' rustdoc - Generate Rust documentation' @echo ' (requires kernel .config)' @echo ' rusttest - Runs the Rust tests' - @echo ' (requires kernel .config)' + @echo ' (requires kernel .config; downloads external repos)' @echo ' rust-analyzer - Generate rust-project.json rust-analyzer support file' @echo ' (requires kernel .config)' @echo '' @@ -1822,11 +1823,21 @@ rusttest: prepare0 # Formatting targets PHONY += rustfmt rustfmtcheck +# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream. +# +# We match using absolute paths since `find` does not resolve them +# when matching, which is a problem when e.g. `srctree` is `..`. +# We `grep` afterwards in order to remove the directory entry itself. rustfmt: - find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT) - -rustfmtcheck: - find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT) --check + $(Q)find $(abs_srctree) -type f -name '*.rs' \ + -o -path $(abs_srctree)/rust/alloc -prune \ + -o -path $(abs_objtree)/rust/test -prune \ + | grep -Fv $(abs_srctree)/rust/alloc \ + | grep -Fv $(abs_objtree)/rust/test \ + | xargs $(RUSTFMT) $(rustfmt_flags) + +rustfmtcheck: rustfmt_flags = --check +rustfmtcheck: rustfmt # IDE support targets PHONY += rust-analyzer diff --git a/rust/Makefile b/rust/Makefile index 0afdf6785497b4..2f0d11c48c272b 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -17,18 +17,27 @@ endif obj-$(CONFIG_RUST) += exports.o -RUSTDOC = rustdoc +ifeq ($(quiet),silent_) +cargo_quiet=-q +rust_test_quiet=-q +rustdoc_test_quiet=--test-args -q +else ifeq ($(quiet),quiet_) +rust_test_quiet=-q +rustdoc_test_quiet=--test-args -q +else +cargo_quiet=--verbose +endif -quiet_cmd_rustdoc = RUSTDOC $(if $(filter --test,$(rustdoc_target_flags)),T, ) $(if $(rustdoc_host),H, ) $< +quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \ $(RUSTDOC) $(if $(rustdoc_host),,$(rustc_cross_flags)) \ - $(filter-out --emit=%, $(rustc_flags)) $(rustc_target_flags) $(rustdoc_target_flags) \ - -L $(objtree)/rust/$(if $(filter --test,$(rustdoc_target_flags)),test/) \ - --output $(objtree)/rust/doc --crate-name $(subst rusttest-,,$(subst rustdoc-,,$@)) \ - -Fmissing-docs @$(objtree)/include/generated/rustc_cfg $< + $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \ + $(rustc_target_flags) -L $(objtree)/rust \ + --output $(objtree)/rust/doc --crate-name $(subst rustdoc-,,$@) \ + @$(objtree)/include/generated/rustc_cfg $< -rustdoc: rustdoc-macros rustdoc-compiler_builtins rustdoc-kernel +rustdoc: rustdoc-macros rustdoc-compiler_builtins rustdoc-alloc rustdoc-kernel rustdoc-macros: private rustdoc_host = yes rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ @@ -39,6 +48,15 @@ rustdoc-macros: $(srctree)/rust/macros/lib.rs FORCE rustdoc-compiler_builtins: $(srctree)/rust/compiler_builtins.rs FORCE $(call if_changed,rustdoc) +# We need to allow `broken_intra_doc_links` because some +# `no_global_oom_handling` functions refer to non-`no_global_oom_handling` +# functions. Ideally `rustdoc` would have a way to distinguish broken links +# due to things that are "configured out" vs. entirely non-existing ones. +rustdoc-alloc: private rustc_target_flags = \ + -Abroken_intra_doc_links +rustdoc-alloc: $(srctree)/rust/alloc/lib.rs FORCE + $(call if_changed,rustdoc) + rustdoc-kernel: private rustc_target_flags = --extern alloc \ --extern build_error \ --extern macros=$(objtree)/rust/libmacros.so @@ -49,50 +67,108 @@ rustdoc-kernel: $(srctree)/rust/kernel/lib.rs rustdoc-macros \ quiet_cmd_rustc_test_library = RUSTC TL $< cmd_rustc_test_library = \ RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \ - $(RUSTC) $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \ + $(RUSTC) $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \ $(rustc_target_flags) --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \ --out-dir $(objtree)/rust/test/ --cfg testlib \ + --sysroot $(objtree)/rust/test/sysroot \ -L $(objtree)/rust/test/ --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< -rusttestlib-build_error: $(srctree)/rust/build_error.rs FORCE +rusttestlib-build_error: $(srctree)/rust/build_error.rs rusttest-prepare FORCE $(call if_changed,rustc_test_library) rusttestlib-macros: private rustc_target_flags = --extern proc_macro rusttestlib-macros: private rustc_test_library_proc = yes -rusttestlib-macros: $(srctree)/rust/macros/lib.rs FORCE +rusttestlib-macros: $(srctree)/rust/macros/lib.rs rusttest-prepare FORCE $(call if_changed,rustc_test_library) +quiet_cmd_rustdoc_test = RUSTDOC T $< + cmd_rustdoc_test = \ + RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \ + $(RUSTDOC) --test $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \ + $(rustc_target_flags) $(rustdoc_test_target_flags) \ + --sysroot $(objtree)/rust/test/sysroot $(rustdoc_test_quiet) \ + -L $(objtree)/rust/test \ + --output $(objtree)/rust/doc --crate-name $(subst rusttest-,,$@) \ + @$(objtree)/include/generated/rustc_cfg $< + # We cannot use `-Zpanic-abort-tests` because some tests are dynamic, # so for the moment we skip `-Cpanic=abort`. quiet_cmd_rustc_test = RUSTC T $< cmd_rustc_test = \ RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \ - $(RUSTC) --test $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \ + $(RUSTC) --test $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \ $(rustc_target_flags) --out-dir $(objtree)/rust/test \ + --sysroot $(objtree)/rust/test/sysroot \ -L $(objtree)/rust/test/ --crate-name $(subst rusttest-,,$@) $<; \ - $(objtree)/rust/test/$(subst rusttest-,,$@) $(rustc_test_run_flags) + $(objtree)/rust/test/$(subst rusttest-,,$@) $(rust_test_quiet) \ + $(rustc_test_run_flags) rusttest: rusttest-macros rusttest-kernel +# This prepares a custom sysroot with our custom `alloc` instead of +# the standard one. +# +# This requires several hacks: +# - Unlike `core` and `alloc`, `std` depends on more than a dozen crates, +# including third-party crates that need to be downloaded, plus custom +# `build.rs` steps. Thus hardcoding things here is not maintainable. +# - `cargo` knows how to build the standard library, but it is an unstable +# feature so far (`-Zbuild-std`). +# - `cargo` only considers the use case of building the standard library +# to use it in a given package. Thus we need to create a dummy package +# and pick the generated libraries from there. +# - Since we only keep a subset of upstream `alloc` in-tree, we need +# to recreate it on the fly by putting our sources on top. +# - The usual ways of modifying the dependency graph in `cargo` do not seem +# to apply for the `-Zbuild-std` steps, thus we have to mislead it +# by modifying the sources in the sysroot. +# - To avoid messing with the user's Rust installation, we create a clone +# of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std` +# steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag. +# +# In the future, we hope to avoid the whole ordeal by either: +# - Making the `test` crate not depend on `std` (either improving upstream +# or having our own custom crate). +# - Making the tests run in kernel space (requires the previous point). +# - Making `std` and friends be more like a "normal" crate, so that +# `-Zbuild-std` and related hacks are not needed. +quiet_cmd_rustsysroot = RUSTSYSROOT + cmd_rustsysroot = \ + rm -rf $(objtree)/rust/test; \ + mkdir -p $(objtree)/rust/test; \ + cp -a $(rustc_sysroot) $(objtree)/rust/test/sysroot; \ + cp -r $(srctree)/rust/alloc/* \ + $(objtree)/rust/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \ + echo '\#!/bin/sh' > $(objtree)/rust/test/rustc_sysroot; \ + echo "$(RUSTC) --sysroot=$(abspath $(objtree)/rust/test/sysroot) \"\$$@\"" \ + >> $(objtree)/rust/test/rustc_sysroot; \ + chmod u+x $(objtree)/rust/test/rustc_sysroot; \ + $(CARGO) -q new $(objtree)/rust/test/dummy; \ + RUSTC=$(objtree)/rust/test/rustc_sysroot $(CARGO) $(cargo_quiet) \ + test -Zbuild-std --target $(rustc_host_target) \ + --manifest-path $(objtree)/rust/test/dummy/Cargo.toml; \ + rm $(objtree)/rust/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \ + cp $(objtree)/rust/test/dummy/target/$(rustc_host_target)/debug/deps/* \ + $(objtree)/rust/test/sysroot/lib/rustlib/$(rustc_host_target)/lib + +rusttest-prepare: FORCE + $(call if_changed,rustsysroot) + rusttest-macros: private rustc_target_flags = --extern proc_macro -rusttest-macros: private rustdoc_host = yes -rusttest-macros: private rustdoc_target_flags = --test --crate-type proc-macro -rusttest-macros: $(srctree)/rust/macros/lib.rs FORCE +rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro +rusttest-macros: $(srctree)/rust/macros/lib.rs rusttest-prepare FORCE $(call if_changed,rustc_test) - $(call if_changed,rustdoc) + $(call if_changed,rustdoc_test) rusttest-kernel: private rustc_target_flags = --extern alloc \ - --extern build_error \ - --extern macros=$(objtree)/rust/test/libmacros.so + --extern build_error --extern macros rusttest-kernel: private rustc_test_run_flags = \ --skip bindgen_test_layout_ -rusttest-kernel: private rustdoc_host = yes -rusttest-kernel: private rustdoc_target_flags = --test -rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttestlib-build_error \ - rusttestlib-macros FORCE +rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttest-prepare \ + rusttestlib-build_error rusttestlib-macros FORCE $(call if_changed,rustc_test) $(call if_changed,rustc_test_library) - $(call if_changed,rustdoc) + $(call if_changed,rustdoc_test) ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) @@ -188,6 +264,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L # `$(rustc_flags)` is passed in case the user added `--sysroot`. rustc_sysroot = $(shell $(RUSTC) $(rustc_flags) --print sysroot) +rustc_host_target = $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2) RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library rust-analyzer: @@ -198,6 +275,11 @@ $(objtree)/rust/compiler_builtins.o: $(srctree)/rust/compiler_builtins.rs \ $(objtree)/rust/core.o FORCE $(call if_changed_dep,rustc_library) +$(objtree)/rust/alloc.o: private skip_clippy = 1 +$(objtree)/rust/alloc.o: $(srctree)/rust/alloc/lib.rs \ + $(objtree)/rust/compiler_builtins.o FORCE + $(call if_changed_dep,rustc_library) + $(objtree)/rust/build_error.o: $(srctree)/rust/build_error.rs \ $(objtree)/rust/compiler_builtins.o FORCE $(call if_changed_dep,rustc_library) @@ -216,8 +298,3 @@ $(objtree)/rust/kernel.o: $(srctree)/rust/kernel/lib.rs $(objtree)/rust/alloc.o $(objtree)/rust/core.o: private skip_clippy = 1 $(objtree)/rust/core.o: $$(RUST_LIB_SRC)/core/src/lib.rs FORCE $(call if_changed_dep,rustc_library) - -$(objtree)/rust/alloc.o: private skip_clippy = 1 -$(objtree)/rust/alloc.o: $$(RUST_LIB_SRC)/alloc/src/lib.rs \ - $(objtree)/rust/compiler_builtins.o FORCE - $(call if_changed_dep,rustc_library)