Merge pull request #1339 from bjorn3/better_compiler_management

Many build system improvements

Author: bjorn3

.github/workflows/main.yml

@@ -62,14 +62,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
@@ -99,9 +91,12 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y gcc-s390x-linux-gnu qemu-user
 
-    - name: Windows setup
-      if: matrix.os == 'windows-latest'
-      run: git config --global core.autocrlf false
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
 
     - name: Prepare dependencies
       run: ./y.rs prepare

.github/workflows/nightly-cranelift.yml

@@ -13,11 +13,15 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Prepare dependencies
+    - name: Use sparse cargo registry
       run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "User"
-        ./y.rs prepare
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
 
     - name: Patch Cranelift
       run: |
@@ -32,6 +36,8 @@ jobs:
 
         cat Cargo.toml
 
+        cargo fetch
+
     - name: Build without unstable features
       # This is the config rust-lang/rust uses for builds
       run: ./y.rs build --no-unstable-features

.github/workflows/rustc.yml

@@ -10,25 +10,21 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
         path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
-    - name: Prepare dependencies
+    - name: Use sparse cargo registry
       run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "User"
-        ./y.rs prepare
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
 
     - name: Test
       run: ./scripts/test_bootstrap.sh
@@ -38,25 +34,21 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
         path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
-    - name: Prepare dependencies
+    - name: Use sparse cargo registry
       run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "User"
-        ./y.rs prepare
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
 
     - name: Test
       run: ./scripts/test_rustc_tests.sh

.vscode/settings.json

@@ -1,4 +1,6 @@
 {
+    "editor.formatOnSave": true,
+
     // source for rustc_* is not included in the rust-src component; disable the errors about this
     "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"],
     "rust-analyzer.imports.granularity.enforce": true,

Readme.md

@@ -8,9 +8,9 @@ If not please open an issue.
 ## Building and testing
 
 ```bash
-$ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git
+$ git clone https://github.com/bjorn3/rustc_codegen_cranelift
 $ cd rustc_codegen_cranelift
-$ ./y.rs prepare # download and patch sysroot src and install hyperfine for benchmarking
+$ ./y.rs prepare
 $ ./y.rs build
 ```
 
@@ -20,13 +20,12 @@ To run the test suite replace the last command with:
 $ ./test.sh
 ```
 
-This will implicitly build cg_clif too. Both `y.rs build` and `test.sh` accept a `--debug` argument to
-build in debug mode.
+For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.rs`.
 
-Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section
+Alternatively you can download a pre built version from [Github Actions]. It is listed in the artifacts section
 of workflow runs. Unfortunately due to GHA restrictions you need to be logged in to access it.
 
-[GHA]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
+[Github Actions]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
 
 ## Usage
 

build_system/abi_cafe.rs

@@ -10,41 +10,36 @@ use super::SysrootKind;
 pub(crate) static ABI_CAFE_REPO: GitRepo =
     GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
 
-static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
+pub(crate) static ABI_CAFE: CargoProject =
+    CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
 
 pub(crate) fn run(
     channel: &str,
     sysroot_kind: SysrootKind,
     dirs: &Dirs,
     cg_clif_dylib: &Path,
-    host_triple: &str,
-    target_triple: &str,
+    host_compiler: &Compiler,
 ) {
     if !config::get_bool("testsuite.abi-cafe") {
         eprintln!("[SKIP] abi-cafe");
         return;
     }
 
-    if host_triple != target_triple {
-        eprintln!("[SKIP] abi-cafe (cross-compilation not supported)");
-        return;
-    }
-
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(
         dirs,
         channel,
         sysroot_kind,
         cg_clif_dylib,
-        host_triple,
-        target_triple,
+        host_compiler,
+        &host_compiler.triple,
     );
 
     eprintln!("Running abi-cafe");
 
     let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];
 
-    let mut cmd = ABI_CAFE.run(&Compiler::host(), dirs);
+    let mut cmd = ABI_CAFE.run(host_compiler, dirs);
     cmd.arg("--");
     cmd.arg("--pairs");
     cmd.args(pairs);

build_system/bench.rs

@@ -21,20 +21,19 @@ pub(crate) static SIMPLE_RAYTRACER_LLVM: CargoProject =
 pub(crate) static SIMPLE_RAYTRACER: CargoProject =
     CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
 
-pub(crate) fn benchmark(dirs: &Dirs) {
-    benchmark_simple_raytracer(dirs);
+pub(crate) fn benchmark(dirs: &Dirs, host_compiler: &Compiler) {
+    benchmark_simple_raytracer(dirs, host_compiler);
 }
 
-fn benchmark_simple_raytracer(dirs: &Dirs) {
+fn benchmark_simple_raytracer(dirs: &Dirs, host_compiler: &Compiler) {
     if std::process::Command::new("hyperfine").output().is_err() {
         eprintln!("Hyperfine not installed");
         eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
         std::process::exit(1);
     }
 
     eprintln!("[LLVM BUILD] simple-raytracer");
-    let host_compiler = Compiler::host();
-    let build_cmd = SIMPLE_RAYTRACER_LLVM.build(&host_compiler, dirs);
+    let build_cmd = SIMPLE_RAYTRACER_LLVM.build(host_compiler, dirs);
     spawn_and_wait(build_cmd);
     fs::copy(
         SIMPLE_RAYTRACER_LLVM

build_system/build_backend.rs

@@ -5,15 +5,15 @@ use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
 use super::utils::{is_ci, CargoProject, Compiler};
 
-static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
+pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
 pub(crate) fn build_backend(
     dirs: &Dirs,
     channel: &str,
-    host_triple: &str,
+    host_compiler: &Compiler,
     use_unstable_features: bool,
 ) -> PathBuf {
-    let mut cmd = CG_CLIF.build(&Compiler::host(), dirs);
+    let mut cmd = CG_CLIF.build(&host_compiler, dirs);
 
     cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
 
@@ -48,7 +48,7 @@ pub(crate) fn build_backend(
 
     CG_CLIF
         .target_dir(dirs)
-        .join(host_triple)
+        .join(&host_compiler.triple)
         .join(channel)
         .join(get_file_name("rustc_codegen_cranelift", "dylib"))
 }

build_system/build_sysroot.rs

@@ -17,7 +17,7 @@ pub(crate) fn build_sysroot(
     channel: &str,
     sysroot_kind: SysrootKind,
     cg_clif_dylib_src: &Path,
-    host_triple: &str,
+    host_compiler: &Compiler,
     target_triple: &str,
 ) {
     eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
@@ -53,7 +53,7 @@ pub(crate) fn build_sysroot(
 
     let default_sysroot = super::rustc_info::get_default_sysroot();
 
-    let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(host_triple).join("lib");
+    let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(&host_compiler.triple).join("lib");
     let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(target_triple).join("lib");
     fs::create_dir_all(&host_rustlib_lib).unwrap();
     fs::create_dir_all(&target_rustlib_lib).unwrap();
@@ -83,7 +83,7 @@ pub(crate) fn build_sysroot(
         SysrootKind::None => {} // Nothing to do
         SysrootKind::Llvm => {
             for file in fs::read_dir(
-                default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"),
+                default_sysroot.join("lib").join("rustlib").join(&host_compiler.triple).join("lib"),
             )
             .unwrap()
             {
@@ -103,7 +103,7 @@ pub(crate) fn build_sysroot(
                 try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
             }
 
-            if target_triple != host_triple {
+            if target_triple != host_compiler.triple {
                 for file in fs::read_dir(
                     default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
                 )
@@ -115,21 +115,24 @@ pub(crate) fn build_sysroot(
             }
         }
         SysrootKind::Clif => {
-            build_clif_sysroot_for_triple(dirs, channel, host_triple, &cg_clif_dylib_path, None);
-
-            if host_triple != target_triple {
-                // When cross-compiling it is often necessary to manually pick the right linker
-                let linker = match target_triple {
-                    "aarch64-unknown-linux-gnu" => Some("aarch64-linux-gnu-gcc"),
-                    "s390x-unknown-linux-gnu" => Some("s390x-linux-gnu-gcc"),
-                    _ => None,
-                };
+            build_clif_sysroot_for_triple(
+                dirs,
+                channel,
+                host_compiler.clone(),
+                &cg_clif_dylib_path,
+            );
+
+            if host_compiler.triple != target_triple {
                 build_clif_sysroot_for_triple(
                     dirs,
                     channel,
-                    target_triple,
+                    {
+                        let mut target_compiler = host_compiler.clone();
+                        target_compiler.triple = target_triple.to_owned();
+                        target_compiler.set_cross_linker_and_runner();
+                        target_compiler
+                    },
                     &cg_clif_dylib_path,
-                    linker,
                 );
             }
 
@@ -150,14 +153,14 @@ pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysr
 pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
 pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
 pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
-static STANDARD_LIBRARY: CargoProject = CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
+pub(crate) static STANDARD_LIBRARY: CargoProject =
+    CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
 
 fn build_clif_sysroot_for_triple(
     dirs: &Dirs,
     channel: &str,
-    triple: &str,
+    mut compiler: Compiler,
     cg_clif_dylib_path: &Path,
-    linker: Option<&str>,
 ) {
     match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
         Err(e) => {
@@ -177,7 +180,7 @@ fn build_clif_sysroot_for_triple(
         }
     }
 
-    let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(triple).join(channel);
+    let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel);
 
     if !super::config::get_bool("keep_sysroot") {
         // Cleanup the deps dir, but keep build scripts and the incremental cache for faster
@@ -188,18 +191,13 @@ fn build_clif_sysroot_for_triple(
     }
 
     // Build sysroot
-    let mut rustflags = "-Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
+    let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
     rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
     rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap()));
     if channel == "release" {
         rustflags.push_str(" -Zmir-opt-level=3");
     }
-    if let Some(linker) = linker {
-        use std::fmt::Write;
-        write!(rustflags, " -Clinker={}", linker).unwrap();
-    }
-    let mut compiler = Compiler::with_triple(triple.to_owned());
-    compiler.rustflags = rustflags;
+    compiler.rustflags += &rustflags;
     let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
     if channel == "release" {
         build_cmd.arg("--release");
@@ -219,7 +217,7 @@ fn build_clif_sysroot_for_triple(
         };
         try_hard_link(
             entry.path(),
-            RUSTLIB_DIR.to_path(dirs).join(triple).join("lib").join(entry.file_name()),
+            RUSTLIB_DIR.to_path(dirs).join(&compiler.triple).join("lib").join(entry.file_name()),
         );
     }
 }