From c2f235b24ff89a8b8b581771ebf01f163511c30f Mon Sep 17 00:00:00 2001 From: jyn Date: Thu, 13 Apr 2023 23:35:43 -0500 Subject: [PATCH 1/2] Allow multiple buildtests to share the same source crate --- Cargo.toml | 1 + tests/buildtest/runner.rs | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e05df74..6f7fc60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,4 +49,5 @@ windows-sys = {version = "0.36.1", features = ["Win32_Foundation", "Win32_System [dev-dependencies] env_logger = "0.8" +rand = "0.8.5" tiny_http = "0.8.0" diff --git a/tests/buildtest/runner.rs b/tests/buildtest/runner.rs index d708b86..eac5a1e 100644 --- a/tests/buildtest/runner.rs +++ b/tests/buildtest/runner.rs @@ -1,4 +1,5 @@ use failure::Error; +use rand::{distributions::Alphanumeric, Rng}; use rustwide::{cmd::SandboxBuilder, Build, BuildBuilder, Crate, Toolchain, Workspace}; use std::path::Path; @@ -40,7 +41,16 @@ impl Runner { sandbox: SandboxBuilder, f: impl FnOnce(BuildBuilder) -> Result, ) -> Result { - let mut dir = self.workspace.build_dir(&self.crate_name); + // Use a random string at the end to avoid conflicts if multiple tests use the same source crate. + let suffix: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .map(char::from) + .collect(); + + let mut dir = self + .workspace + .build_dir(&format!("{}-{suffix}", &self.crate_name)); dir.purge()?; f(dir.build(&self.toolchain, &self.krate, sandbox)) } From 399d37cb2c9044217a54e9f5c5b826041c5d8089 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 27 Nov 2021 14:55:50 -0500 Subject: [PATCH 2/2] Fetch dependencies for `-Zbuild-std` before entering the sandbox This allows running `doc -Zbuild-std` from within the sandbox. Previously, it would error out because cargo tried to download the standard library's dependencies: ``` [2021-11-27T19:57:24Z INFO rustwide::cmd] running `Command { std: "docker" "create" "-v" "/home/joshua/src/rust/docs.rs/.workspace/builds/gba-0.5.2/target:/opt/rustwide/target:rw,Z" "-v" "/home/joshua/src/rust/docs.rs/.workspace/builds/gba-0.5.2/source:/opt/rustwide/workdir:ro,Z" "-v" "/home/joshua/src/rust/docs.rs/.workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/home/joshua/src/rust/docs.rs/.workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "DOCS_RS=1" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "3221225472" "--user" "1000:1000" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux-micro" "/opt/rustwide/cargo-home/bin/cargo" "+nightly" "rustdoc" "--lib" "-Zrustdoc-map" "-Z" "unstable-options" "--config" "build.rustdocflags=[\"--cfg\", \"docs_rs\", \"-Z\", \"unstable-options\", \"--emit=invocation-specific\", \"--resource-suffix\", \"-20211126-1.58.0-nightly-6d246f0c8\", \"--static-root-path\", \"/\", \"--cap-lints\", \"warn\", \"--disable-per-crate-search\"]" "-Zunstable-options" "--config=doc.extern-map.registries.crates-io=\"https://docs.rs/{pkg_name}/{version}/thumbv4t-none-eabi\"" "-Zbuild-std" "--target" "thumbv4t-none-eabi", kill_on_drop: false }` [2021-11-27T19:57:24Z INFO rustwide::cmd] [stdout] fe2773f8a17ab13ce7b66c7221f91883a7b92b3bdeef7b30bf2ed55aa6b3d511 [2021-11-27T19:57:24Z INFO rustwide::cmd] running `Command { std: "docker" "start" "-a" "fe2773f8a17ab13ce7b66c7221f91883a7b92b3bdeef7b30bf2ed55aa6b3d511", kill_on_drop: false }` [2021-11-27T19:57:24Z INFO rustwide::cmd] [stderr] Downloading crates ... [2021-11-27T19:57:24Z INFO rustwide::cmd] [stderr] warning: spurious network error (2 tries remaining): [6] Couldn't resolve host name (Could not resolve host: crates.io) [2021-11-27T19:57:24Z INFO rustwide::cmd] [stderr] error: failed to download from `https://crates.io/api/v1/crates/libc/0.2.106/download` [2021-11-27T19:57:24Z INFO rustwide::cmd] [stderr] [2021-11-27T19:57:24Z INFO rustwide::cmd] [stderr] Caused by: [2021-11-27T19:57:24Z INFO rustwide::cmd] [stderr] [6] Couldn't resolve host name (Could not resolve host: crates.io) ``` --- CHANGELOG.md | 6 ++++++ src/build.rs | 16 ++++++++++++++ src/prepare.rs | 49 +++++++++++++++++++++++++++++------------- tests/buildtest/mod.rs | 31 ++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6911157..c4001fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- New method `Build::fetch_build_std_dependencies` for using `-Zbuild-std` within the sandbox when + networking is disabled. Previously, this would try to fetch the standard library sources, which + would error when networking was blocked. + ## [0.15.2] - 2022-11-08 ### Changed diff --git a/src/build.rs b/src/build.rs index 7989df9..0a23625 100644 --- a/src/build.rs +++ b/src/build.rs @@ -314,4 +314,20 @@ impl<'ws> Build<'ws> { pub fn host_target_dir(&self) -> PathBuf { self.dir.target_dir() } + + /// Pre-fetching the dependencies for `-Z build-std` outside the sandbox. + /// + /// When this function is called, it is possible to use `-Zbuild-std` inside + /// the sandbox to build the standard library from source even when + /// networking is disabled. + #[cfg(any(feature = "unstable", doc))] + #[cfg_attr(docs_rs, doc(cfg(feature = "unstable")))] + pub fn fetch_build_std_dependencies(&self, targets: &[&str]) -> Result<(), Error> { + crate::prepare::fetch_deps( + &self.dir.workspace, + self.toolchain, + &self.host_source_dir(), + targets, + ) + } } diff --git a/src/prepare.rs b/src/prepare.rs index e76b363..402db27 100644 --- a/src/prepare.rs +++ b/src/prepare.rs @@ -138,21 +138,40 @@ impl<'a> Prepare<'a> { } fn fetch_deps(&mut self) -> Result<(), Error> { - let mut missing_deps = false; - let res = Command::new(self.workspace, self.toolchain.cargo()) - .args(&["fetch", "--manifest-path", "Cargo.toml"]) - .cd(self.source_dir) - .process_lines(&mut |line, _| { - if line.contains("failed to load source for dependency") { - missing_deps = true; - } - }) - .run(); - match res { - Ok(_) => Ok(()), - Err(_) if missing_deps => Err(PrepareError::MissingDependencies.into()), - err => err.map_err(|e| e.into()), - } + fetch_deps(self.workspace, self.toolchain, self.source_dir, &[]) + } +} + +pub(crate) fn fetch_deps( + workspace: &Workspace, + toolchain: &Toolchain, + source_dir: &Path, + fetch_build_std_targets: &[&str], +) -> Result<(), Error> { + let mut missing_deps = false; + let mut cmd = Command::new(workspace, toolchain.cargo()) + .args(&["fetch", "--manifest-path", "Cargo.toml"]) + .cd(source_dir); + // Pass `-Zbuild-std` in case a build in the sandbox wants to use it; + // build-std has to have the source for libstd's dependencies available. + if !fetch_build_std_targets.is_empty() { + toolchain.add_component(workspace, "rust-src")?; + cmd = cmd.args(&["-Zbuild-std"]).env("RUSTC_BOOTSTRAP", "1"); + } + for target in fetch_build_std_targets { + cmd = cmd.args(&["--target", target]); + } + let res = cmd + .process_lines(&mut |line, _| { + if line.contains("failed to load source for dependency") { + missing_deps = true; + } + }) + .run(); + match res { + Ok(_) => Ok(()), + Err(_) if missing_deps => Err(PrepareError::MissingDependencies.into()), + err => err.map_err(|e| e.into()), } } diff --git a/tests/buildtest/mod.rs b/tests/buildtest/mod.rs index 15de460..b99b1cb 100644 --- a/tests/buildtest/mod.rs +++ b/tests/buildtest/mod.rs @@ -26,6 +26,37 @@ fn test_hello_world() { }); } +#[test] +#[cfg(feature = "unstable")] +fn test_fetch_build_std() { + use std::path::Path; + + let target_file = Path::new(env!("OUT_DIR")).join("target"); + let target = std::fs::read_to_string(target_file).unwrap(); + + runner::run("hello-world", |run| { + run.run(SandboxBuilder::new().enable_networking(false), |build| { + build.fetch_build_std_dependencies(&vec![target.as_str()])?; + let storage = rustwide::logging::LogStorage::new(LevelFilter::Info); + rustwide::logging::capture(&storage, || -> Result<_, Error> { + build + .cargo() + .env("RUSTC_BOOTSTRAP", "1") + .args(&["run", "-Zbuild-std", "--target", &target]) + .run()?; + Ok(()) + })?; + + assert!(storage.to_string().contains("[stdout] Hello, world!\n")); + assert!(storage + .to_string() + .contains("[stdout] Hello, world again!\n")); + Ok(()) + })?; + Ok(()) + }); +} + #[test] fn path_based_patch() { runner::run("path-based-patch", |run| {