From 6ef6987bb376a671349f8318e3b4b5c092f0523b Mon Sep 17 00:00:00 2001 From: David Crespo Date: Wed, 29 Oct 2025 16:34:04 -0500 Subject: [PATCH 1/3] claude first pass --- Cargo.lock | 1 + dev-tools/downloader/Cargo.toml | 1 + dev-tools/downloader/src/lib.rs | 151 ++++++++++++++++++++++++++++++-- 3 files changed, 146 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 478961381a8..ba045d16901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16390,6 +16390,7 @@ dependencies = [ "slog-term", "strum 0.27.2", "tar", + "tempfile", "tokio", ] diff --git a/dev-tools/downloader/Cargo.toml b/dev-tools/downloader/Cargo.toml index 9f64a2a6e0c..95adffc281b 100644 --- a/dev-tools/downloader/Cargo.toml +++ b/dev-tools/downloader/Cargo.toml @@ -21,4 +21,5 @@ slog-async.workspace = true slog-term.workspace = true strum.workspace = true tar.workspace = true +tempfile.workspace = true tokio.workspace = true diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index ace32d61252..b960a8ba52a 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -216,6 +216,107 @@ impl<'a> Downloader<'a> { ) -> Self { Self { log, output_dir, versions_dir } } + + /// Build a binary from a git repository at a specific commit. + /// + /// This function: + /// 1. Checks for cached binaries at `out/.build-cache/{project}/{commit}/` + /// 2. If not cached, shallow clones the repo to a temp directory + /// 3. Builds the specified binaries with cargo + /// 4. Caches the built binaries + /// 5. Returns paths to the cached binaries + async fn build_from_git( + &self, + project: &str, + repo_url: &str, + commit: &str, + binaries: &[(&str, &[&str])], // (binary_name, cargo_args) + ) -> Result> { + let cache_dir = + self.output_dir.join(".build-cache").join(project).join(commit); + + // Check if all binaries are already cached + let mut cached_paths = Vec::new(); + let mut all_cached = true; + for (binary_name, _) in binaries { + let cached_path = cache_dir.join(binary_name); + if !cached_path.exists() { + all_cached = false; + break; + } + cached_paths.push(cached_path); + } + + if all_cached { + info!(self.log, "Found cached binaries for {project} at {commit}"; "cache_dir" => %cache_dir); + return Ok(cached_paths); + } + + // Need to build - create temp directory + info!(self.log, "Building {project} from source at commit {commit}"); + let temp_dir = tempfile::tempdir()?; + let temp_path = Utf8PathBuf::try_from(temp_dir.path().to_path_buf())?; + + // Shallow clone the repository + info!(self.log, "Cloning {repo_url}"); + let mut clone_cmd = Command::new("git"); + clone_cmd + .arg("clone") + .arg("--depth") + .arg("1") + .arg("--branch") + .arg(commit) + .arg(repo_url) + .arg(&temp_path); + + let clone_output = clone_cmd.output().await?; + if !clone_output.status.success() { + let stderr = String::from_utf8_lossy(&clone_output.stderr); + bail!("Failed to clone {repo_url}: {stderr}"); + } + + // Build each binary + tokio::fs::create_dir_all(&cache_dir).await?; + let mut result_paths = Vec::new(); + + for (binary_name, cargo_args) in binaries { + info!(self.log, "Building {binary_name}"; "args" => ?cargo_args); + + let mut build_cmd = Command::new("cargo"); + build_cmd + .arg("build") + .arg("--bin") + .arg(binary_name) + .args(*cargo_args) + .current_dir(&temp_path); + + let build_output = build_cmd.output().await?; + if !build_output.status.success() { + let stderr = String::from_utf8_lossy(&build_output.stderr); + bail!("Failed to build {binary_name}: {stderr}"); + } + + // Determine source path based on whether --release was used + let is_release = cargo_args.contains(&"--release"); + let profile = if is_release { "release" } else { "debug" }; + let source_path = + temp_path.join("target").join(profile).join(binary_name); + + if !source_path.exists() { + bail!("Expected binary not found at {source_path}"); + } + + // Copy to cache + let cached_path = cache_dir.join(binary_name); + tokio::fs::copy(&source_path, &cached_path).await?; + set_permissions(&cached_path, 0o755).await?; + + result_paths.push(cached_path); + } + + info!(self.log, "Successfully built and cached {project} binaries"); + Ok(result_paths) + } } /// Parses a file of the format: @@ -738,13 +839,30 @@ impl Downloader<'_> { } Os::Illumos => {} Os::Mac => { - warn!(self.log, "WARNING: Dendrite not available for Mac"); - warn!(self.log, "Network APIs will be unavailable"); - - let path = bin_dir.join("dpd"); - tokio::fs::write(&path, "echo 'unsupported os' && exit 1") + info!(self.log, "Building dendrite from source for macOS"); + + let binaries = [ + ("dpd", &["--release", "--features=tofino_stub"][..]), + ("swadm", &["--release"][..]), + ]; + + let built_binaries = self + .build_from_git( + "dendrite", + "https://github.com/oxidecomputer/dendrite", + &commit, + &binaries, + ) .await?; - set_permissions(&path, 0o755).await?; + + // Copy built binaries to bin_dir + for (binary_path, (binary_name, _)) in + built_binaries.iter().zip(binaries.iter()) + { + let dest = bin_dir.join(binary_name); + tokio::fs::copy(binary_path, &dest).await?; + set_permissions(&dest, 0o755).await?; + } } } @@ -807,7 +925,26 @@ impl Downloader<'_> { set_permissions(&path, 0o755).await?; tokio::fs::copy(path, binary_dir.join(filename)).await?; } - _ => (), + Os::Mac => { + info!(self.log, "Building maghemite from source for macOS"); + + let binaries = [("mgd", &["--no-default-features"][..])]; + + let built_binaries = self + .build_from_git( + "maghemite", + "https://github.com/oxidecomputer/maghemite", + &commit, + &binaries, + ) + .await?; + + // Copy built binary to binary_dir + let dest = binary_dir.join("mgd"); + tokio::fs::copy(&built_binaries[0], &dest).await?; + set_permissions(&dest, 0o755).await?; + } + Os::Illumos => (), } Ok(()) From c4ae1cdcd0aa5a97dfe2cba306f48cc4180ab312 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Wed, 29 Oct 2025 16:36:35 -0500 Subject: [PATCH 2/3] better clone technique and unsupported OS messages --- dev-tools/downloader/src/lib.rs | 18 +++++++++++++----- tools/install_builder_prerequisites.sh | 2 +- tools/install_runner_prerequisites.sh | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index b960a8ba52a..b00345aa960 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -257,15 +257,12 @@ impl<'a> Downloader<'a> { let temp_dir = tempfile::tempdir()?; let temp_path = Utf8PathBuf::try_from(temp_dir.path().to_path_buf())?; - // Shallow clone the repository + // Clone and checkout the specific commit info!(self.log, "Cloning {repo_url}"); let mut clone_cmd = Command::new("git"); clone_cmd .arg("clone") - .arg("--depth") - .arg("1") - .arg("--branch") - .arg(commit) + .arg("--filter=blob:none") .arg(repo_url) .arg(&temp_path); @@ -275,6 +272,17 @@ impl<'a> Downloader<'a> { bail!("Failed to clone {repo_url}: {stderr}"); } + // Checkout the specific commit + info!(self.log, "Checking out commit {commit}"); + let mut checkout_cmd = Command::new("git"); + checkout_cmd.arg("checkout").arg(commit).current_dir(&temp_path); + + let checkout_output = checkout_cmd.output().await?; + if !checkout_output.status.success() { + let stderr = String::from_utf8_lossy(&checkout_output.stderr); + bail!("Failed to checkout {commit}: {stderr}"); + } + // Build each binary tokio::fs::create_dir_all(&cache_dir).await?; let mut result_paths = Vec::new(); diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 486bf0b33da..646c764f8e9 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -199,7 +199,7 @@ function install_packages { ) confirm "Install (or update) [${packages[*]}]?" && brew install "${packages[@]}" else - echo "Unsupported OS: ${HOST_OS}" + echo "Skipping builder prereqs for unsupported OS: ${HOST_OS}" exit 1 fi } diff --git a/tools/install_runner_prerequisites.sh b/tools/install_runner_prerequisites.sh index c93f5cdb473..af438257f92 100755 --- a/tools/install_runner_prerequisites.sh +++ b/tools/install_runner_prerequisites.sh @@ -153,7 +153,7 @@ function install_packages { confirm "Install (or update) [${packages[*]}]?" && $maybe_sudo apt-get install "${packages[@]}" fi else - echo "Unsupported OS: ${HOST_OS}" + echo "Skipping runner prereqs for unsupported OS: ${HOST_OS}" exit 1 fi } From e767e5cc3646c489542bbf286dc95d340e251e76 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Thu, 30 Oct 2025 10:04:30 -0500 Subject: [PATCH 3/3] cleanup, always build in release mode --- dev-tools/downloader/src/lib.rs | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index b00345aa960..44fb340de28 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -228,7 +228,6 @@ impl<'a> Downloader<'a> { async fn build_from_git( &self, project: &str, - repo_url: &str, commit: &str, binaries: &[(&str, &[&str])], // (binary_name, cargo_args) ) -> Result> { @@ -258,12 +257,13 @@ impl<'a> Downloader<'a> { let temp_path = Utf8PathBuf::try_from(temp_dir.path().to_path_buf())?; // Clone and checkout the specific commit + let repo_url = format!("https://github.com/oxidecomputer/{}", project); info!(self.log, "Cloning {repo_url}"); let mut clone_cmd = Command::new("git"); clone_cmd .arg("clone") .arg("--filter=blob:none") - .arg(repo_url) + .arg(&repo_url) .arg(&temp_path); let clone_output = clone_cmd.output().await?; @@ -293,6 +293,7 @@ impl<'a> Downloader<'a> { let mut build_cmd = Command::new("cargo"); build_cmd .arg("build") + .arg("--release") .arg("--bin") .arg(binary_name) .args(*cargo_args) @@ -304,11 +305,9 @@ impl<'a> Downloader<'a> { bail!("Failed to build {binary_name}: {stderr}"); } - // Determine source path based on whether --release was used - let is_release = cargo_args.contains(&"--release"); - let profile = if is_release { "release" } else { "debug" }; + // Always build in release mode let source_path = - temp_path.join("target").join(profile).join(binary_name); + temp_path.join("target").join("release").join(binary_name); if !source_path.exists() { bail!("Expected binary not found at {source_path}"); @@ -850,18 +849,12 @@ impl Downloader<'_> { info!(self.log, "Building dendrite from source for macOS"); let binaries = [ - ("dpd", &["--release", "--features=tofino_stub"][..]), - ("swadm", &["--release"][..]), + ("dpd", &["--features=tofino_stub"][..]), + ("swadm", &[][..]), ]; - let built_binaries = self - .build_from_git( - "dendrite", - "https://github.com/oxidecomputer/dendrite", - &commit, - &binaries, - ) - .await?; + let built_binaries = + self.build_from_git("dendrite", &commit, &binaries).await?; // Copy built binaries to bin_dir for (binary_path, (binary_name, _)) in @@ -939,12 +932,7 @@ impl Downloader<'_> { let binaries = [("mgd", &["--no-default-features"][..])]; let built_binaries = self - .build_from_git( - "maghemite", - "https://github.com/oxidecomputer/maghemite", - &commit, - &binaries, - ) + .build_from_git("maghemite", &commit, &binaries) .await?; // Copy built binary to binary_dir