From 1e77b1cea74b42e5a64092aade74c0e8529e3543 Mon Sep 17 00:00:00 2001 From: default <216188+jdx@users.noreply.github.com> Date: Sat, 13 Jun 2026 00:11:21 +0000 Subject: [PATCH] feat(system): build source formulae natively with a formula DSL shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Formulae without a usable bottle (source-only formulae, or bottles that only exist for other platforms) previously failed. mise now builds them from source — still without Homebrew installed: - ruby is provisioned through mise's own tool machinery (precompiled by default), so no interpreter needs to be present on the machine - the formula's .rb is downloaded from homebrew/core pinned to the commit the API metadata was generated from and verified against the API's sha256; the source archive is verified the same way - build dependencies are added to the install closure and poured as regular bottles before the build - shim.rb implements the commonly-used subset of the Formula DSL (configure/cmake/meson builds, resources, patches, path and ENV helpers, std_*_args, completions) and runs def install against the canonical prefix; unsupported install-time helpers fail with a clear error instead of miscompiling silently - the keg gets the same brew-compatible receipt as a poured bottle with poured_from_bottle: false, matching how brew marks source builds The system package manager trait is now async_trait(?Send) because the toolset machinery holds non-Send shell state across awaits; the driver awaits managers sequentially so the futures never cross threads. MISE_SYSTEM_BREW_FORCE_SOURCE= (undocumented, like MISE_SYSTEM_BREW_PREFIX) forces the source path for bottled formulae, which the new e2e test uses to build GNU hello without root. Co-Authored-By: Claude Fable 5 --- docs/system-packages/brew.md | 48 +- e2e/cli/test_system_install_brew_source_slow | 34 + src/system/packages/apt.rs | 2 +- src/system/packages/brew/api.rs | 48 + src/system/packages/brew/mod.rs | 120 ++- src/system/packages/brew/pour.rs | 13 +- src/system/packages/brew/resolve.rs | 33 +- src/system/packages/brew/shim.rb | 910 +++++++++++++++++++ src/system/packages/brew/source.rs | 518 +++++++++++ src/system/packages/dnf.rs | 2 +- src/system/packages/mod.rs | 6 +- src/system/packages/pacman.rs | 2 +- 12 files changed, 1679 insertions(+), 57 deletions(-) create mode 100644 e2e/cli/test_system_install_brew_source_slow create mode 100644 src/system/packages/brew/shim.rb create mode 100644 src/system/packages/brew/source.rs diff --git a/docs/system-packages/brew.md b/docs/system-packages/brew.md index d21ee7a3a8..0bd801664b 100644 --- a/docs/system-packages/brew.md +++ b/docs/system-packages/brew.md @@ -15,7 +15,9 @@ into the canonical Homebrew prefix — `/opt/homebrew` on arm64 macOS, formulae.brew.sh API, resolves the runtime dependency closure, downloads prebuilt bottles from ghcr.io (verifying sha256 checksums), and performs the same relocation, code-signing, and linking work `brew` does when pouring a -bottle. mise never shells out to `brew`. +bottle. Formulae without a usable bottle are built from source, also without +Homebrew (see [Source formulae](#source-formulae)). mise never shells out to +`brew`. This exists because shared-library packages — postgres, ffmpeg, imagemagick, php — fundamentally can't be served by mise's per-project backends like @@ -32,8 +34,9 @@ what makes them work. | Linux arm64 | `/home/linuxbrew/.linuxbrew` | Intel macs are not supported — the `brew` manager reports itself unavailable -there. On Linux, formulae without a bottle for your architecture fail with a -clear error (arm64 Linux bottles exist for most but not all of homebrew/core). +there. On Linux, formulae without a bottle for your architecture (arm64 +Linux bottles exist for most but not all of homebrew/core) are built from +source instead. ## The prefix @@ -82,6 +85,39 @@ For each formula in the dependency closure (dependencies first): [keg-only](https://docs.brew.sh/FAQ#what-does-keg-only-mean) formulae get the `opt` link but are not linked into the prefix, same as brew. +## Source formulae + +A few formulae have no bottle at all (source-only formulae), and some have +bottles for other platforms but not yours. mise builds those from source — +still without Homebrew: + +1. **Ruby** — a formula is Ruby code, so mise provisions a mise-managed + ruby through its normal tool machinery (precompiled, fast; respects your + configured ruby if you have one). +2. **Formula** — the formula's `.rb` is downloaded from homebrew/core, + pinned to the exact commit the API metadata was generated from and + verified against the API's sha256 for it. +3. **Source** — the stable source archive is downloaded and verified + against the API's sha256. +4. **Build deps** — the formula's build dependencies (cmake, pkgconf, ...) + are added to the install closure and poured as regular bottles first. +5. **Build** — mise evaluates the formula with its own Formula-DSL shim and + runs `def install` against the canonical prefix, with `PATH`, + `PKG_CONFIG_PATH`, and compiler flags pointing at the dependency kegs. + The keg gets the same brew-compatible receipt as a poured bottle, with + `poured_from_bottle: false` — exactly how brew marks its own source + builds. + +The shim implements the commonly-used subset of the formula DSL +(configure/cmake/meson-style builds, resources, patches, the standard path +and environment helpers). Formulae that use parts of the DSL the shim +doesn't cover — language-specific helpers like `virtualenv_install_with_resources`, +VCS downloads, and similar — fail with a clear `formula uses ...` error +rather than miscompiling silently. + +Source builds need a working toolchain (Xcode Command Line Tools on macOS, +gcc/make on Linux), exactly as they would under plain Homebrew. + ## Upgrades `mise system upgrade` re-resolves the configured formulae against the @@ -97,8 +133,10 @@ operation. implemented. - **No taps.** Third-party taps are Ruby code that requires Homebrew to evaluate; only homebrew/core is supported. -- **No source builds.** Formulae without a bottle for your platform fail - with a clear error. +- **Source builds cover the common formula shapes.** mise's formula shim + implements the widely-used subset of the DSL (see + [Source formulae](#source-formulae)); formulae that reach beyond it fail + with a clear error naming the unsupported feature. - **Use canonical formula names.** `postgresql@17` is a formula name, not a mise version pin — the API's current stable version decides what gets installed. Aliases (`postgres`) install correctly but `mise system status` diff --git a/e2e/cli/test_system_install_brew_source_slow b/e2e/cli/test_system_install_brew_source_slow new file mode 100644 index 0000000000..4a68525ef0 --- /dev/null +++ b/e2e/cli/test_system_install_brew_source_slow @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Source-build pipeline test: builds GNU hello from source with mise's +# formula shim (mise-managed ruby, no Homebrew installed). Uses a throwaway +# prefix, so no root is needed. Requires network and a C toolchain. +if [[ "$(uname)" != "Linux" ]] || ! command -v cc >/dev/null 2>&1; then + exit 0 +fi +case "$(uname -m)" in + x86_64 | aarch64) ;; + *) exit 0 ;; +esac + +export MISE_SYSTEM_BREW_PREFIX="$HOME/brew-source-prefix" +# hello has bottles everywhere; force it down the source-build path +export MISE_SYSTEM_BREW_FORCE_SOURCE=hello + +cat <mise.toml +[system.packages] +"brew:hello" = "latest" +EOF + +assert_contains "mise system status" "missing" +assert_contains "mise system install --dry-run" "build hello/" + +mise system install --yes +assert_contains "mise system status" "installed" + +# the built binary runs from the prefix and the keg carries a receipt +assert_contains "$MISE_SYSTEM_BREW_PREFIX/bin/hello --greeting=mise" "mise" +assert_contains "cat $MISE_SYSTEM_BREW_PREFIX/Cellar/hello/*/INSTALL_RECEIPT.json" '"poured_from_bottle":false' + +# idempotent +assert_contains "mise system install --yes 2>&1" "already" diff --git a/src/system/packages/apt.rs b/src/system/packages/apt.rs index a1999f7735..c3afafa47c 100644 --- a/src/system/packages/apt.rs +++ b/src/system/packages/apt.rs @@ -101,7 +101,7 @@ fn parse_dpkg_query(output: &str, requests: &[PackageRequest]) -> Vec &'static str { "apt" diff --git a/src/system/packages/brew/api.rs b/src/system/packages/brew/api.rs index 8e2e6fe8a3..d291b023d1 100644 --- a/src/system/packages/brew/api.rs +++ b/src/system/packages/brew/api.rs @@ -23,11 +23,42 @@ pub struct Formula { /// runtime dependencies (formula names) #[serde(default)] pub dependencies: Vec, + /// build-time-only dependencies — needed for source builds, not pours + #[serde(default)] + pub build_dependencies: Vec, #[serde(default)] pub bottle: HashMap, /// per-bottle-tag overrides (e.g. different dependencies on some platforms) #[serde(default)] pub variations: HashMap, + /// source download specs keyed by spec name ("stable") + #[serde(default)] + pub urls: HashMap, + /// formula .rb location in homebrew/core (e.g. "Formula/h/hello.rb") + #[serde(default)] + pub ruby_source_path: Option, + #[serde(default)] + pub ruby_source_checksum: Option, + /// homebrew/core commit this API snapshot was generated from + #[serde(default)] + pub tap_git_head: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SourceUrl { + pub url: String, + /// sha256 of the source archive; absent for VCS sources + #[serde(default)] + pub checksum: Option, + /// non-default download strategy (":git", ":svn", ...) — unsupported + #[serde(default)] + pub using: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct RubySourceChecksum { + #[serde(default)] + pub sha256: Option, } #[derive(Debug, Clone, Deserialize)] @@ -53,6 +84,8 @@ pub struct BottleFile { pub struct Variation { #[serde(default)] pub dependencies: Option>, + #[serde(default)] + pub build_dependencies: Option>, } impl Formula { @@ -80,9 +113,24 @@ impl Formula { &self.dependencies } + /// build-time dependencies for the given bottle tag, applying `variations` + pub fn build_dependencies_for(&self, tag: &str) -> &[String] { + if let Some(v) = self.variations.get(tag) + && let Some(deps) = &v.build_dependencies + { + return deps; + } + &self.build_dependencies + } + pub fn bottle_files(&self) -> Option<&HashMap> { self.bottle.get("stable").map(|b| &b.files) } + + /// the stable source archive spec, when present + pub fn stable_url(&self) -> Option<&SourceUrl> { + self.urls.get("stable") + } } /// Fetch formula metadata by name (or alias — brew's API redirects aliases diff --git a/src/system/packages/brew/mod.rs b/src/system/packages/brew/mod.rs index 7ec85c3336..7896c6f60f 100644 --- a/src/system/packages/brew/mod.rs +++ b/src/system/packages/brew/mod.rs @@ -4,11 +4,16 @@ //! (/opt/homebrew on arm64 macOS, /home/linuxbrew/.linuxbrew on Linux) — //! fetching metadata from formulae.brew.sh, downloading bottles from //! ghcr.io, and doing the same relocation/codesigning work `brew` does at -//! pour time. mise never shells out to brew; the receipts it writes are -//! brew-compatible, so a real Homebrew sees mise-poured kegs as its own. +//! pour time. mise never shells out to brew to pour a bottle; the receipts +//! it writes are brew-compatible, so a real Homebrew sees mise-poured kegs +//! as its own. //! -//! Scope: formulae only. Taps require evaluating Ruby and are unsupported; -//! casks and services are not implemented. +//! Formulae without a usable bottle are built from source, still without +//! Homebrew: mise provisions a mise-managed ruby and evaluates the formula +//! with its own Formula-DSL shim (see source.rs and shim.rb). +//! +//! Scope: formulae only. Casks and services are not implemented; taps are +//! unsupported (only homebrew/core formulae are served by the API). use async_trait::async_trait; use eyre::bail; @@ -26,6 +31,7 @@ mod pour; mod prefix; mod relocate; mod resolve; +mod source; mod state; mod tag; @@ -72,19 +78,38 @@ impl BrewManager { info!("brew: all formulae already poured"); return Ok(()); } + // formulae without a usable bottle are built from source by + // evaluating their Ruby with mise's formula shim; reject the ones + // the builder can't handle before any work happens + let source_builds: Vec<_> = to_pour + .iter() + .filter(|rf| !source::has_bottle(&rf.formula)) + .collect(); + for rf in &source_builds { + source::check_buildable(&rf.formula)?; + } if opts.dry_run { prefix::bootstrap(true)?; for rf in &to_pour { - miseprintln!( - "pour {}/{} ({})", - rf.formula.name, - rf.formula.pkg_version()?, - if rf.on_request { - "requested" - } else { - "dependency" - }, - ); + let origin = if rf.on_request { + "requested" + } else { + "dependency" + }; + if source::has_bottle(&rf.formula) { + miseprintln!( + "pour {}/{} ({origin})", + rf.formula.name, + rf.formula.pkg_version()?, + ); + } else { + miseprintln!( + "build {}/{} from source ({origin}, {})", + rf.formula.name, + rf.formula.pkg_version()?, + source::missing_bottle_reason(&rf.formula), + ); + } } return Ok(()); } @@ -97,41 +122,68 @@ impl BrewManager { } prefix::bootstrap(false)?; prefix::setup_linux_runtime()?; + if !source_builds.is_empty() { + info!( + "brew: building from source (no bottle for this machine): {}", + source_builds + .iter() + .map(|rf| rf.formula.name.clone()) + .collect::>() + .join(", "), + ); + } let mpr = MultiProgressReport::get(); + // overall [cur/total] header above the per-formula clx jobs, same as + // tool installs (no-op when only one formula is being installed) + mpr.init_footer(false, "install", to_pour.len()); let mut ledger = state::Ledger::load(); for rf in &to_pour { let name = &rf.formula.name; let pkg_version = rf.formula.pkg_version()?; - let Some(files) = rf.formula.bottle_files() else { - bail!("{name} has no bottles (source-only formulae are unsupported)"); + let pr: Box = mpr.add(&format!("brew:{name}")); + // branch on the same predicate the upfront classification used + let bottle = if source::has_bottle(&rf.formula) { + rf.formula.bottle_files().and_then(tag::select) + } else { + None }; - let Some((tag, bottle)) = tag::select(files) else { - bail!( - "{name} has no bottle for this machine (available: {})", - files.keys().cloned().collect::>().join(", "), - ); + let installed = match bottle { + Some((tag, bottle)) => { + async { + let tarball = + fetch::fetch_bottle(name, &pkg_version, bottle, Some(&*pr)).await?; + pour::pour(rf, &tag, bottle, &tarball, &closure, &*pr).await?; + Ok(pkg_version.clone()) + } + .await + } + None => source::build(rf, &closure, &*pr) + .await + .map(|()| pkg_version.clone()), }; - let pr: Box = mpr.add(&format!("brew:{name}")); - let poured = async { - let tarball = fetch::fetch_bottle(name, &pkg_version, bottle, Some(&*pr)).await?; - pour::pour(rf, &tag, bottle, &tarball, &closure, &*pr).await - } - .await; - if let Err(err) = poured { - pr.finish_with_icon("failed".to_string(), ProgressIcon::Error); - return Err(err); - } - ledger.record(name, &pkg_version, rf.on_request); + let version = match installed { + Ok(version) => version, + Err(err) => { + pr.finish_with_icon("failed".to_string(), ProgressIcon::Error); + // render the final progress state so the error that + // propagates from here isn't masked by live jobs + mpr.footer_finish(); + return Err(err); + } + }; + ledger.record(name, &version, rf.on_request); ledger.save()?; - pr.finish_with_message(pkg_version.clone()); + pr.finish_with_message(version); + mpr.footer_inc(1); } + mpr.footer_finish(); // a glibc poured in this run repoints /lib/ld.so at it prefix::setup_linux_runtime()?; Ok(()) } } -#[async_trait] +#[async_trait(?Send)] impl SystemPackageManager for BrewManager { fn name(&self) -> &'static str { "brew" diff --git a/src/system/packages/brew/pour.rs b/src/system/packages/brew/pour.rs index c188b82306..d37d7972eb 100644 --- a/src/system/packages/brew/pour.rs +++ b/src/system/packages/brew/pour.rs @@ -128,7 +128,7 @@ pub async fn pour( .wrap_err_with(|| format!("failed to re-sign relocated binaries for {name}"))?; } - write_receipt(rf, tag, &tmp, &report, closure)?; + write_receipt(rf, tag, &tmp, &report, closure, true)?; pr.set_message("link".to_string()); if keg.exists() { @@ -179,13 +179,16 @@ fn bottled_by_homebrew_at_least(keg: &Path, min: (u64, u64, u64)) -> bool { } /// brew-compatible INSTALL_RECEIPT.json so a later-installed real Homebrew -/// adopts these kegs (brew list/upgrade/uninstall all work) -fn write_receipt( +/// adopts these kegs (brew list/upgrade/uninstall all work). Written for +/// both poured bottles and source-built kegs; `poured_from_bottle` +/// distinguishes them the same way brew's own tab does. +pub fn write_receipt( rf: &ResolvedFormula, tag: &str, keg: &Path, report: &relocate::RelocationReport, closure: &[ResolvedFormula], + poured_from_bottle: bool, ) -> Result<()> { let runtime_dependencies: Vec = closure .iter() @@ -219,8 +222,8 @@ fn write_receipt( "homebrew_version": "5.1.15 (mise)", "used_options": [], "unused_options": [], - "built_as_bottle": true, - "poured_from_bottle": true, + "built_as_bottle": poured_from_bottle, + "poured_from_bottle": poured_from_bottle, "loaded_from_api": true, "installed_as_dependency": !rf.on_request, "installed_on_request": rf.on_request, diff --git a/src/system/packages/brew/resolve.rs b/src/system/packages/brew/resolve.rs index e5fe6307c4..62619323ff 100644 --- a/src/system/packages/brew/resolve.rs +++ b/src/system/packages/brew/resolve.rs @@ -15,13 +15,28 @@ pub struct ResolvedFormula { pub on_request: bool, } -/// the `variations` entry that applies to the bottle that will actually be -/// poured — the selected bottle tag, which may be older than the host's -fn dep_tag(formula: &Formula, host_tag: &str) -> String { - formula - .bottle_files() - .and_then(|files| tag::select(files).map(|(tag, _)| tag)) - .unwrap_or_else(|| host_tag.to_string()) +/// The `variations` entry that applies to what will actually be installed: +/// the selected bottle tag (which may be older than the host's), or the +/// host's own tag for formulae that will be built from source. Shared with +/// source.rs so the build environment walks the same dependency lists this +/// resolution installed. +pub fn dep_tag(formula: &Formula, host_tag: &str) -> String { + if super::source::has_bottle(formula) + && let Some((tag, _)) = formula.bottle_files().and_then(tag::select) + { + return tag; + } + host_tag.to_string() +} + +/// dependencies that must be installed before this formula: runtime deps +/// always, plus build deps when the formula will be built from source +fn install_deps<'a>(formula: &'a Formula, tag: &str) -> Vec<&'a String> { + let mut deps: Vec<&String> = formula.dependencies_for(tag).iter().collect(); + if !super::source::has_bottle(formula) { + deps.extend(formula.build_dependencies_for(tag)); + } + deps } /// Resolve the runtime closure of `roots` into install order (dependencies @@ -49,7 +64,7 @@ pub async fn resolve_closure(roots: &[String]) -> Result> { } if !formulae.contains_key(&c) { let tag = dep_tag(&formula, &host_tag); - for dep in formula.dependencies_for(&tag) { + for dep in install_deps(&formula, &tag) { queue.push((dep.clone(), false)); } formulae.insert(c.clone(), formula); @@ -91,7 +106,7 @@ pub async fn resolve_closure(roots: &[String]) -> Result> { }; visiting.push(name.to_string()); let tag = dep_tag(formula, host_tag); - for dep in formula.dependencies_for(&tag) { + for dep in install_deps(formula, &tag) { let dep_name = canonical.get(dep).cloned().unwrap_or_else(|| dep.clone()); visit( &dep_name, host_tag, formulae, canonical, done, visiting, on_request, sorted, diff --git a/src/system/packages/brew/shim.rb b/src/system/packages/brew/shim.rb new file mode 100644 index 0000000000..bde6349157 --- /dev/null +++ b/src/system/packages/brew/shim.rb @@ -0,0 +1,910 @@ +# frozen_string_literal: true + +# mise's Homebrew formula build shim. +# +# Evaluates a homebrew/core formula .rb file and runs its `install` method +# against mise's prefix layout, without Homebrew installed. This implements +# the commonly-used subset of the Formula DSL; formulae that reach outside it +# fail loudly with a clear message rather than miscompiling silently. +# +# Contract with the Rust side (src/system/packages/brew/source.rs), all via +# environment variables: +# MISE_BREW_PREFIX Homebrew prefix (/opt/homebrew, ...) +# MISE_BREW_CELLAR /Cellar +# MISE_BREW_FORMULA_FILE path to the formula .rb (sha-verified by Rust) +# MISE_BREW_NAME canonical formula name +# MISE_BREW_VERSION upstream version ("2.12.3") +# MISE_BREW_PKG_VERSION keg directory name ("2.12.3_1") +# MISE_BREW_BUILDPATH staged source directory (also the cwd) +# MISE_BREW_CACHE download cache for resources/patches +# MISE_BREW_MAKE_JOBS build parallelism +# +# The main source archive is downloaded, verified, and staged by Rust before +# this script runs; the shim only downloads resources and external patches +# (each verified against the sha256 declared in the formula). + +require "digest/sha2" +require "etc" +require "fileutils" +require "open-uri" +require "open3" +require "pathname" +require "rbconfig" +require "shellwords" +require "tmpdir" + +MISE_BREW_PREFIX = Pathname.new(ENV.fetch("MISE_BREW_PREFIX")) +MISE_BREW_CELLAR = Pathname.new(ENV.fetch("MISE_BREW_CELLAR")) +MISE_BREW_FORMULA_FILE = Pathname.new(ENV.fetch("MISE_BREW_FORMULA_FILE")) +MISE_BREW_NAME = ENV.fetch("MISE_BREW_NAME") +MISE_BREW_VERSION = ENV.fetch("MISE_BREW_VERSION") +MISE_BREW_PKG_VERSION = ENV.fetch("MISE_BREW_PKG_VERSION") +MISE_BREW_BUILDPATH = Pathname.new(ENV.fetch("MISE_BREW_BUILDPATH")) +MISE_BREW_CACHE = Pathname.new(ENV.fetch("MISE_BREW_CACHE")) +MISE_BREW_MAKE_JOBS = ENV.fetch("MISE_BREW_MAKE_JOBS", "4") + +def ohai(*args) + $stdout.puts "==> #{args.join(" ")}" + $stdout.flush +end + +def opoo(message) + $stderr.puts "Warning: #{message}" +end + +def odie(message) + $stderr.puts "Error: #{message}" + exit 1 +end + +class ShimUnsupportedError < StandardError; end + +def shim_unsupported!(feature) + raise ShimUnsupportedError, + "formula uses `#{feature}`, which mise's source-build shim does not support" +end + +module MiseDownload + module_function + + # download with redirects into the cache, verify, return the path + def fetch(url, sha256, context) + raise "#{context}: missing sha256" if sha256.to_s.strip.empty? + sha256 = sha256.to_s.strip.downcase + raise "#{context}: malformed sha256" unless sha256.match?(/\A[0-9a-f]{64}\z/) + + MISE_BREW_CACHE.mkpath + dest = MISE_BREW_CACHE + "#{sha256}--#{File.basename(URI(url).path)}" + unless dest.file? && Digest::SHA256.file(dest).hexdigest == sha256 + ohai "Downloading #{url}" + tmp = Pathname.new("#{dest}.incomplete") + URI.open(url, "rb", redirect: true) do |remote| + tmp.open("wb") { |f| IO.copy_stream(remote, f) } + end + actual = Digest::SHA256.file(tmp).hexdigest + if actual != sha256 + tmp.unlink + raise "#{context}: sha256 mismatch (expected #{sha256}, got #{actual})" + end + tmp.rename(dest) + end + dest + end + + # unpack an archive the way brew stages sources: if the archive contains a + # single top-level directory, its contents become the stage root + def unpack(archive, dest) + dest.mkpath + case archive.basename.to_s + when /\.(tar\.(gz|xz|bz2|zst)|tgz|txz|tbz2?|tar|crate)\z/i + system_or_die "tar", "xf", archive.to_s, "-C", dest.to_s + when /\.zip\z/i + system_or_die "unzip", "-qo", archive.to_s, "-d", dest.to_s + when /\.(gz|xz|bz2)\z/i + data = `#{archive.to_s =~ /xz\z/ ? "xz -dc" : archive.to_s =~ /bz2\z/ ? "bzip2 -dc" : "gzip -dc"} #{Shellwords.escape(archive.to_s)}` + raise "failed to decompress #{archive}" unless $?.success? + (dest + archive.basename.to_s.sub(/\.(gz|xz|bz2)\z/i, "")).binwrite(data) + else + FileUtils.cp archive, dest + end + entries = dest.children + return entries.first if entries.size == 1 && entries.first.directory? + dest + end + + def system_or_die(*args) + raise "command failed: #{args.join(" ")}" unless system(*args) + end +end + +module OS + def self.mac? = RbConfig::CONFIG["host_os"].include?("darwin") + def self.linux? = RbConfig::CONFIG["host_os"].include?("linux") + + module Mac + def self.version = MacOSVersion.host + end + + module Linux + def self.languages = ["en"] + end +end + +class MacOSVersion + include Comparable + SYMBOLS = { + tahoe: "26", sequoia: "15", sonoma: "14", ventura: "13", + monterey: "12", big_sur: "11", catalina: "10.15", mojave: "10.14", + high_sierra: "10.13", sierra: "10.12", el_capitan: "10.11", + }.freeze + + def self.host + @host ||= begin + version = OS.mac? ? `sw_vers -productVersion`.strip : "0" + new(version.empty? ? "0" : version) + end + end + + def self.from_symbol(sym) = new(SYMBOLS.fetch(sym.to_sym, "0")) + + def initialize(version) = @version = version.to_s + + def <=>(other) + other = self.class.from_symbol(other) if other.is_a?(Symbol) + other = self.class.new(other.to_s) unless other.is_a?(MacOSVersion) + Gem::Version.new(@version) <=> Gem::Version.new(other.to_s) + end + + def to_s = @version + def major = @version.split(".").first.to_i + def requires_nehalem_cpu? = false +end + +module Hardware + module CPU + def self.arch = RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/ ? :arm64 : :x86_64 + def self.arm? = arch == :arm64 + def self.intel? = arch == :x86_64 + def self.is_64_bit? = true + def self.cores = Etc.respond_to?(:nprocessors) ? Etc.nprocessors : 4 + end +end + +module MacOS + def self.version = MacOSVersion.host + + def self.sdk_path + @sdk_path ||= Pathname.new(`xcrun --show-sdk-path 2>/dev/null`.strip) + end + + def self.sdk_path_if_needed = OS.mac? ? sdk_path : nil + + module CLT + def self.installed? = OS.mac? && File.directory?("/Library/Developer/CommandLineTools") + end + + module Xcode + def self.installed? = false + end +end + +class Version + include Comparable + + def initialize(version) = @version = version.to_s + + def <=>(other) = Gem::Version.new(@version.gsub(/[^0-9.].*\z/, "")) <=> Gem::Version.new(other.to_s.gsub(/[^0-9.].*\z/, "")) + def to_s = @version + def to_str = @version + def inspect = @version.inspect + + def major = token(0) + def minor = token(1) + def patch = token(2) + def major_minor = Version.new(@version.split(".")[0, 2].to_a.join(".")) + def major_minor_patch = Version.new(@version.split(".")[0, 3].to_a.join(".")) + def csv = @version.split(",").map { |part| Version.new(part) } + + private + + def token(idx) + part = @version.split(".")[idx] + part.nil? ? nil : Version.new(part) + end +end + +# Pathname extensions mirroring brew's +class Pathname + def install(*sources) + mkpath + sources.flatten.each do |src| + case src + when Hash + src.each { |from, to| install_one(from, self + to) } + else + install_one(src, self + Pathname.new(src.to_s).basename) + end + end + end + + def install_one(src, dest) + src = Pathname.new(src.to_s) + raise "cannot install #{src}: does not exist" unless src.exist? || src.symlink? + dest.dirname.mkpath + FileUtils.mv src.to_s, dest.to_s + end + private :install_one + + def install_symlink(*sources) + mkpath + sources.flatten.each do |src| + case src + when Hash + src.each { |from, to| make_relative_symlink(Pathname.new(from.to_s), self + to) } + else + src = Pathname.new(src.to_s) + make_relative_symlink(src, self + src.basename) + end + end + end + + def make_relative_symlink(target, link) + link.dirname.mkpath + link.unlink if link.symlink? || link.file? + link.make_symlink(target.relative_path_from(link.dirname)) + end + private :make_relative_symlink + + def install_metafiles(from = Pathname.pwd) + mkpath + Pathname.new(from).children.each do |child| + next unless child.file? + next unless child.basename.to_s =~ /\A(readme|license|licence|copying|copyright|news|changelog|changes|authors)(\.|\z)/i + FileUtils.cp child, self + end + end + + def write(content, *args) + dirname.mkpath + super + end + + def atomic_write(content) + dirname.mkpath + File.write(to_s, content) + end + + def append_lines(content) + open("a") { |f| f.puts(content) } + end + + def ensure_executable! + chmod(0o755) if file? + end +end + +class BuildOptions + def with?(_name) = false + def without?(name) = !with?(name) + def head? = false + def stable? = true + def bottle? = false + def include?(_name) = false + def used_options = [] + def unused_options = [] +end + +# brew's build-environment helpers, grafted onto the real ENV object the way +# brew's EnvActivation does — formulae call `ENV.append`, `ENV.cc`, etc. and +# constant lookup always resolves `ENV` to Object::ENV, so the global must +# carry the methods itself +module BrewEnvExtension + KNOWN_NOOPS = %i[ + permit_arch_flags runtime_cpu_detection O0 O1 O2 O3 Os + cxx11 libcxx no_fixup_chains deverbose_build refurbish_args + permit_weak_imports + ].freeze + + def append(keys, value, separator = " ") + Array(keys).each do |key| + old = self[key.to_s] + self[key.to_s] = old.nil? || old.empty? ? value.to_s : "#{old}#{separator}#{value}" + end + end + + def prepend(keys, value, separator = " ") + Array(keys).each do |key| + old = self[key.to_s] + self[key.to_s] = old.nil? || old.empty? ? value.to_s : "#{value}#{separator}#{old}" + end + end + + def append_path(key, path) = append(key, path, File::PATH_SEPARATOR) + def prepend_path(key, path) = prepend(key, path, File::PATH_SEPARATOR) + + def prepend_create_path(key, path) + Pathname.new(path.to_s).mkpath + prepend_path(key, path) + end + + def remove(keys, value) + Array(keys).each do |key| + next if self[key.to_s].nil? + self[key.to_s] = self[key.to_s].sub(value, "").strip + end + end + + def append_to_cflags(flag) = append(%w[CFLAGS CXXFLAGS], flag) + def remove_from_cflags(flag) = remove(%w[CFLAGS CXXFLAGS], flag) + + def cc = fetch("CC", "cc") + def cxx = fetch("CXX", "c++") + def cflags = self["CFLAGS"] + def cxxflags = self["CXXFLAGS"] + def cppflags = self["CPPFLAGS"] + def ldflags = self["LDFLAGS"] + + def make_jobs = MISE_BREW_MAKE_JOBS.to_i + + def deparallelize + old = delete("MAKEFLAGS") + if block_given? + begin + yield + ensure + self["MAKEFLAGS"] = old unless old.nil? + end + end + old + end + + def method_missing(name, *_args) + unless KNOWN_NOOPS.include?(name) + opoo "ENV.#{name} is not supported by mise's build shim (ignored)" + end + nil + end + + def respond_to_missing?(_name, _include_private = false) = true +end + +ENV.extend(BrewEnvExtension) + +class Resource + attr_reader :name + attr_accessor :owner + + def initialize(name) + @name = name + @specs = {} + end + + def url(url = nil, **specs) + @url = url unless url.nil? + @specs.merge!(specs) + @url + end + + def sha256(sha = nil) + @sha256 = sha unless sha.nil? + @sha256 + end + + def version(version = nil) + @version = version unless version.nil? + @version || (@url && Version.new(@url[/[0-9]+(?:\.[0-9]+)+/].to_s)) + end + + def mirror(url) = (@mirrors ||= []) << url + def using = @specs[:using] + def livecheck(&) = nil + + def patch(*, &) + # building against an unpatched resource silently produces a wrong + # artifact — refuse instead + shim_unsupported!("resource patches") + end + + # unpack into `target` (or a tmpdir) and run the optional block there + def stage(target = nil, &block) + shim_unsupported!("resource with download strategy #{using.inspect}") unless using.nil? + raise "resource #{name}: missing url" if @url.nil? + archive = MiseDownload.fetch(@url, @sha256, "resource #{name}") + if target + target = Pathname.new(target.to_s) + stage_dir = Pathname.new(Dir.mktmpdir("mise-resource-")) + root = MiseDownload.unpack(archive, stage_dir) + target.mkpath + FileUtils.cp_r("#{root}/.", target.to_s) + FileUtils.remove_entry(stage_dir) + block&.call(self) + else + raise "resource #{name}: stage requires a target or a block" unless block + Dir.mktmpdir("mise-resource-") do |dir| + root = MiseDownload.unpack(archive, Pathname.new(dir)) + Dir.chdir(root) { block.call(self) } + end + end + end +end + +class PatchSpec + def initialize(strip, formula_file) + @strip = strip + @formula_file = formula_file + end + + def url(url = nil, **) + @url = url unless url.nil? + @url + end + + def sha256(sha = nil) + @sha256 = sha unless sha.nil? + @sha256 + end + + def data! + @data = true + end + + def apply! + if @data + content = @formula_file.read.split(/^__END__$/, 2)[1] + raise "formula declares a DATA patch but has no __END__ section" if content.nil? + apply_content(content.sub(/\A\n/, "")) + elsif @url + file = MiseDownload.fetch(@url, @sha256, "patch") + apply_content(file.read) + end + end + + private + + def apply_content(content) + ohai "Applying patch (-p#{@strip})" + Open3.popen2e("patch", "-g", "0", "-f", "-p#{@strip}") do |stdin, out, thread| + stdin.write(content) + stdin.close + output = out.read + raise "patch failed:\n#{output}" unless thread.value.success? + $stdout.puts output + end + end +end + +# reference to an installed dependency, as returned by Formula["name"] +class DependencyFormula + def initialize(name) = @name = name + + def opt_prefix = MISE_BREW_PREFIX + "opt" + @name + def opt_bin = opt_prefix + "bin" + def opt_lib = opt_prefix + "lib" + def opt_include = opt_prefix + "include" + def opt_libexec = opt_prefix + "libexec" + def opt_share = opt_prefix + "share" + def opt_frameworks = opt_prefix + "Frameworks" + + # resolved keg path (through the opt symlink) + def prefix = opt_prefix.exist? ? opt_prefix.realpath : opt_prefix + def bin = prefix + "bin" + def lib = prefix + "lib" + def include = prefix + "include" + def libexec = prefix + "libexec" + def share = prefix + "share" + + def installed? = opt_prefix.exist? + def any_installed? = installed? + + def version + Version.new(prefix.basename.to_s.sub(/_\d+\z/, "")) + rescue StandardError + Version.new("0") + end + + def name = @name + def to_s = @name +end + +class Formula + include FileUtils + + class << self + def inherited(subclass) + super + Formula.instance_variable_set(:@formula_subclass, subclass) + end + + def [](name) = DependencyFormula.new(name.to_s) + + # ---- recorded-but-inert metadata DSL ---- + def desc(*); end + def homepage(*); end + def license(*); end + def revision(*); end + def version_scheme(*); end + def compatibility_version(*); end + def no_autobump!(*, **); end + def mirror(*); end + + def url(url = nil, **) + @url = url + end + + def sha256(*args) + # top-level stable sha; bottle-block shas never reach here because the + # bottle block is not evaluated + end + + def version(v = nil) + @explicit_version = v unless v.nil? + end + + # blocks that must not be evaluated + def bottle(&) = nil + def head(*, &) = nil + def livecheck(&) = nil + def service(&) = nil + def test(&) = nil + def plist_options(*); end + + def stable(&block) + class_exec(&block) if block + end + + def depends_on(*); end + def uses_from_macos(*, **); end + def keg_only(*); end + def skip_clean(*); end + def link_overwrite(*); end + def conflicts_with(*, **); end + def fails_with(*, &) = nil + def needs(*); end + def env(*); end + def option(*, **); end + def deprecated_option(*); end + def pour_bottle?(*, &) = nil + def allow_network_access!(*); end + def deny_network_access!(*); end + + def deprecate!(**kwargs) + opoo "formula is deprecated upstream#{kwargs[:because] ? " (#{kwargs[:because]})" : ""}" + end + + def disable!(**kwargs) + opoo "formula is disabled upstream#{kwargs[:because] ? " (#{kwargs[:because]})" : ""}" + end + + # ---- platform-conditional blocks ---- + def on_macos(&block) + class_exec(&block) if OS.mac? && block + end + + def on_linux(&block) + class_exec(&block) if OS.linux? && block + end + + def on_arm(&block) + class_exec(&block) if Hardware::CPU.arm? && block + end + + def on_intel(&block) + class_exec(&block) if Hardware::CPU.intel? && block + end + + # `on_system :linux, macos: :ventura_or_older` — run on linux, or on + # macOS when the host version matches the comparator + def on_system(*conditions, macos: nil, &block) + run = conditions.include?(:linux) && OS.linux? + run ||= OS.mac? && macos && macos_condition_matches?(macos) + class_exec(&block) if run && block + end + + def macos_condition_matches?(condition) + sym = condition.to_s + base, comparator = if sym.end_with?("_or_older") + [sym.delete_suffix("_or_older"), :or_older] + elsif sym.end_with?("_or_newer") + [sym.delete_suffix("_or_newer"), :or_newer] + else + [sym, :==] + end + unless MacOSVersion::SYMBOLS.key?(base.to_sym) + # an unknown version symbol must not silently skip install logic + shim_unsupported!("on_system macos condition #{condition.inspect}") + end + host = MacOSVersion.host + target = MacOSVersion.from_symbol(base.to_sym) + case comparator + when :or_older then host <= target + when :or_newer then host >= target + else host.major == target.major + end + end + private :macos_condition_matches? + + # macOS-version blocks (on_sonoma, on_ventura :or_older, ...) + MacOSVersion::SYMBOLS.each_key do |sym| + define_method(:"on_#{sym}") do |comparator = :==, &block| + next unless OS.mac? && block + host = MacOSVersion.host + target = MacOSVersion.from_symbol(sym) + run = case comparator + when :or_older then host <= target + when :or_newer then host >= target + else host.major == target.major + end + class_exec(&block) if run + end + end + + # ---- resources & patches ---- + def resource(name, &block) + @resources ||= {} + res = Resource.new(name) + res.instance_eval(&block) if block + @resources[name] = res + end + + def resources = (@resources ||= {}) + + def patch(strip = :p1, src = nil, &block) + @patches ||= [] + strip, src = :p1, strip if strip == :DATA || strip.is_a?(String) + spec = PatchSpec.new(strip.to_s.delete_prefix("p"), MISE_BREW_FORMULA_FILE) + if src == :DATA + spec.data! + elsif src.is_a?(String) + # inline patch string is unsupported; brew core doesn't use it + shim_unsupported!("inline patch strings") + end + spec.instance_eval(&block) if block + @patches << spec + end + + def patches = (@patches ||= []) + + # Unknown class-level DSL is almost always newly-added inert metadata + # (livecheck variants, autobump markers, ...) — formula files track + # brew's current DSL, so failing here would break working formulae every + # time brew adds an annotation. Warn so it stays visible. Install-time + # helpers (instance methods) still fail loudly: those shape the build. + def method_missing(name, *_args, &_block) + opoo "ignoring unknown formula DSL `#{name}` (mise build shim)" + nil + end + + def respond_to_missing?(_name, _include_private = false) = true + end + + # ---- instance API used inside def install ---- + def name = MISE_BREW_NAME + def version = Version.new(MISE_BREW_VERSION) + def pkg_version = MISE_BREW_PKG_VERSION + def build = BuildOptions.new + def head? = false + def stable? = true + + def prefix = MISE_BREW_CELLAR + name + MISE_BREW_PKG_VERSION + def opt_prefix = MISE_BREW_PREFIX + "opt" + name + def opt_bin = opt_prefix + "bin" + def opt_lib = opt_prefix + "lib" + def opt_libexec = opt_prefix + "libexec" + def opt_share = opt_prefix + "share" + + def bin = prefix + "bin" + def sbin = prefix + "sbin" + def lib = prefix + "lib" + def libexec = prefix + "libexec" + def include = prefix + "include" + def frameworks = prefix + "Frameworks" + def share = prefix + "share" + def pkgshare = share + name + def elisp = share + "emacs/site-lisp" + name + def man = share + "man" + (1..8).each { |n| define_method(:"man#{n}") { man + "man#{n}" } } + def doc = share + "doc" + name + def info = share + "info" + def bash_completion = prefix + "etc/bash_completion.d" + def zsh_completion = share + "zsh/site-functions" + def fish_completion = share + "fish/vendor_completions.d" + def pkgetc = etc + name + + # etc/var live in the prefix, outside the keg (survive upgrades) + def etc = MISE_BREW_PREFIX + "etc" + def var = MISE_BREW_PREFIX + "var" + + def buildpath = MISE_BREW_BUILDPATH + def testpath = shim_unsupported!("testpath") + + def deps = [] + def declared_deps = [] + + def resource(name) + res = self.class.resources.fetch(name) { raise "undefined resource #{name.inspect}" } + res.owner = self + res + end + + def resources = self.class.resources.values.each { |r| r.owner = self } + + # ---- build helpers ---- + def system(cmd, *args) + args = args.map(&:to_s) + pretty = ([cmd] + args).join(" ") + ohai pretty + # single-string invocations go through the shell, like Kernel#system + ok = Kernel.system(cmd.to_s, *args) + raise "command failed: #{pretty}" unless ok + end + + def quiet_system(cmd, *args) + Kernel.system(cmd.to_s, *args.map(&:to_s), out: File::NULL, err: File::NULL) + end + + def which(cmd) + ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).each do |dir| + candidate = Pathname.new(dir) + cmd.to_s + return candidate if candidate.executable? && candidate.file? + end + nil + end + + def inreplace(paths, before = nil, after = nil, audit_result = true, &block) + Array(paths).each do |path| + path = Pathname.new(path.to_s) + content = path.read + replaced = content.dup + if block + ext = InreplaceText.new(replaced) + block.call(ext) + replaced = ext.text + else + raise "inreplace: missing before/after" if before.nil? + case before + when Regexp then replaced.gsub!(before, after.to_s) + else replaced = replaced.gsub(before.to_s, after.to_s) + end + end + if audit_result && replaced == content + raise "inreplace in #{path} made no substitutions — the formula may need updating" + end + path.write(replaced) + end + end + + def cd(path, &block) = Dir.chdir(path.to_s, &block) + + def mkdir(name, &block) + path = Pathname.new(name.to_s) + path.mkpath + block ? Dir.chdir(path.to_s, &block) : path + end + + def loader_path = OS.mac? ? "@loader_path" : "$ORIGIN" + + def rpath(source: bin, target: lib) + "#{loader_path}/#{Pathname.new(target.to_s).relative_path_from(Pathname.new(source.to_s))}" + end + + def std_configure_args + [ + "--disable-debug", + "--disable-dependency-tracking", + "--disable-silent-rules", + "--prefix=#{prefix}", + "--libdir=#{lib}", + ] + end + + def std_cmake_args(install_prefix: prefix, install_libdir: "lib", find_framework: "LAST") + [ + "-DCMAKE_INSTALL_PREFIX=#{install_prefix}", + "-DCMAKE_INSTALL_LIBDIR=#{install_libdir}", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_FIND_FRAMEWORK=#{find_framework}", + "-DCMAKE_VERBOSE_MAKEFILE=ON", + "-DBUILD_TESTING=OFF", + "-Wno-dev", + ] + end + + def std_meson_args + ["--prefix=#{prefix}", "--libdir=#{lib}", "--buildtype=release", "--wrap-mode=nofallback"] + end + + def std_cargo_args(root: prefix, path: ".") + ["--jobs", ENV["MAKEFLAGS"].to_s[/-j(\d+)/, 1] || MISE_BREW_MAKE_JOBS, "--locked", "--root", root.to_s, "--path", path.to_s].tap(&:compact!) + end + + def std_go_args(ldflags: nil, output: bin/name, tags: nil) + args = ["-trimpath", "-o=#{output}"] + args += ["-tags=#{Array(tags).join(",")}"] if tags + args += ["-ldflags=#{Array(ldflags).join(" ")}"] if ldflags + args + end + + def generate_completions_from_executable(*commands, base_name: name, shells: [:bash, :zsh, :fish], shell_parameter_format: nil) + shells.each do |shell| + completion_dir = { bash: bash_completion, zsh: zsh_completion, fish: fish_completion }.fetch(shell) + file_name = { bash: base_name, zsh: "_#{base_name}", fish: "#{base_name}.fish" }.fetch(shell) + shell_arg = case shell_parameter_format + when nil then shell.to_s + when :flag then "--#{shell}" + when :arg then "--shell=#{shell}" + when :none then nil + when :click then nil # click uses env vars; handled below + when String then "#{shell_parameter_format}#{shell}" + end + cmd = commands.map(&:to_s) + cmd << shell_arg unless shell_arg.nil? + env = {} + if shell_parameter_format == :click + env["_#{base_name.upcase.tr("-", "_")}_COMPLETE"] = "#{shell}_source" + end + output, status = Open3.capture2(env, *cmd) + raise "completion generation failed: #{cmd.join(" ")}" unless status.success? + completion_dir.mkpath + (completion_dir + file_name).write(output) + end + end + + def time = Time.now + + def post_install; end + + # unknown instance helpers: fail loudly + def method_missing(name, *args, &block) + shim_unsupported!("#{name} (install-time helper)") + end + + def respond_to_missing?(_name, _include_private = false) = true + + class InreplaceText + attr_reader :text + + def initialize(text) = @text = text + + def gsub!(before, after, audit_result = true) + result = @text.gsub!(before, after.to_s) + raise "inreplace: #{before.inspect} not found" if result.nil? && audit_result + result + end + + def sub!(before, after) + @text.sub!(before, after.to_s) + end + + def change_make_var!(flag, new_value) + replaced = @text.gsub!(/^#{Regexp.escape(flag)}[ \t]*[\\?\\+\\:]?=[ \t]*((?:.*\\\n)*.*)$/, "#{flag}=#{new_value}") + opoo "change_make_var! #{flag} did nothing" if replaced.nil? + replaced + end + end +end + +def main + load MISE_BREW_FORMULA_FILE.to_s + klass = Formula.instance_variable_get(:@formula_subclass) + odie "no Formula subclass found in #{MISE_BREW_FORMULA_FILE}" if klass.nil? + formula = klass.new + + Dir.chdir(MISE_BREW_BUILDPATH.to_s) + klass.patches.each(&:apply!) + + ohai "#{MISE_BREW_NAME}: running install" + formula.prefix.mkpath + formula.install + formula.post_install + + if formula.prefix.children.empty? + odie "install completed but the keg at #{formula.prefix} is empty" + end +rescue ShimUnsupportedError => e + odie e.message +rescue StandardError => e + $stderr.puts e.backtrace.first(10).map { |l| " #{l}" }.join("\n") if ENV["MISE_DEBUG"] + odie "build failed: #{e.message}" +end + +main diff --git a/src/system/packages/brew/source.rs b/src/system/packages/brew/source.rs new file mode 100644 index 0000000000..e45a194745 --- /dev/null +++ b/src/system/packages/brew/source.rs @@ -0,0 +1,518 @@ +//! Native source builds for formulae without a usable bottle. +//! +//! Building a formula means running its Ruby `install` method. mise does +//! this without Homebrew: it provisions a mise-managed ruby (precompiled, +//! via the normal tool machinery), downloads the formula's .rb from +//! homebrew/core (sha256-verified against the API metadata), stages the +//! sha256-verified source archive, and evaluates the formula with the +//! Formula-DSL shim in shim.rb. Build dependencies are poured as bottles +//! beforehand by the regular closure machinery (see resolve.rs), so the +//! build environment points at real kegs in the canonical prefix. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use eyre::{WrapErr, bail}; + +use super::api::Formula; +use super::pour; +use super::prefix; +use super::resolve::ResolvedFormula; +use super::tag; +use crate::cmd::CmdLineRunner; +use crate::config::{Config, Settings}; +use crate::file::{ExtractOptions, TarFormat}; +use crate::http::HTTP_FETCH; +use crate::result::Result; +use crate::toolset::{InstallOptions, ToolsetBuilder}; +use crate::ui::progress_report::SingleReport; + +const SHIM_RB: &str = include_str!("shim.rb"); +const HOMEBREW_CORE_RAW: &str = "https://raw.githubusercontent.com/Homebrew/homebrew-core"; + +/// does this formula have a bottle that can be poured on this machine? +pub fn has_bottle(formula: &Formula) -> bool { + // undocumented override for testing the source-build pipeline with + // formulae that do have bottles (comma-separated names) + if let Ok(force) = crate::env::var("MISE_SYSTEM_BREW_FORCE_SOURCE") + && force.split(',').any(|f| f.trim() == formula.name) + { + return false; + } + formula + .bottle_files() + .and_then(|files| tag::select(files)) + .is_some() +} + +/// why `has_bottle` is false, for log/dry-run output +pub fn missing_bottle_reason(formula: &Formula) -> String { + match formula.bottle_files() { + Some(files) if !files.is_empty() => { + let mut tags: Vec = files.keys().cloned().collect(); + tags.sort(); + format!("bottles exist only for: {}", tags.join(", ")) + } + _ => "source-only formula, no bottles".to_string(), + } +} + +/// Reject early what the source builder cannot handle, with the reason — +/// checked before any work happens so dry-run and real runs fail alike. +pub fn check_buildable(formula: &Formula) -> Result<()> { + let Some(src) = formula.stable_url() else { + bail!("{}: formula has no stable source URL", formula.name); + }; + if let Some(using) = &src.using { + bail!( + "{}: source uses the {using:?} download strategy, which mise cannot build from \ + (and no bottle exists for this machine)", + formula.name, + ); + } + if src.checksum.is_none() { + bail!("{}: source archive has no sha256 in the API", formula.name); + } + // the formula .rb must be pinned to the API snapshot's commit and + // verifiable — evaluating a newer/unverified formula against older + // source metadata would build the wrong thing + if formula.ruby_source_path.is_none() { + bail!("{}: API metadata has no ruby_source_path", formula.name); + } + if formula.tap_git_head.is_none() { + bail!("{}: API metadata has no tap_git_head", formula.name); + } + if formula + .ruby_source_checksum + .as_ref() + .and_then(|c| c.sha256.as_deref()) + .is_none() + { + bail!("{}: API metadata has no formula checksum", formula.name); + } + Ok(()) +} + +/// Build a formula from source into its keg and link it. +pub async fn build( + rf: &ResolvedFormula, + closure: &[ResolvedFormula], + pr: &dyn SingleReport, +) -> Result<()> { + let formula = &rf.formula; + let name = &formula.name; + let pkg_version = formula.pkg_version()?; + check_buildable(formula)?; + + pr.set_message("resolve ruby".to_string()); + let ruby = ruby_bin().await?; + let formula_rb = fetch_formula_rb(formula, pr).await?; + let archive = fetch_source(formula, pr).await?; + + let build_root = crate::dirs::CACHE + .join("system-brew") + .join("build") + .join(format!("{name}-{pkg_version}")); + if build_root.exists() { + crate::file::remove_all(&build_root)?; + } + crate::file::create_dir_all(&build_root)?; + pr.set_message("extract source".to_string()); + let buildpath = stage_source(&archive, &build_root, &source_basename(formula))?; + let shim_path = build_root.join("mise-brew-shim.rb"); + crate::file::write(&shim_path, SHIM_RB)?; + + // formulae bake the final keg path into binaries, so the build installs + // straight into the Cellar (same as brew); a failed build removes the keg + let keg = pour::keg_path(name, &pkg_version); + if keg.exists() { + crate::file::remove_all(&keg)?; + } + + pr.set_message("build from source".to_string()); + let cmd = CmdLineRunner::new(&ruby) + .arg(&shim_path) + .current_dir(&buildpath) + .envs(build_env( + rf, + closure, + &pkg_version, + &buildpath, + &formula_rb, + )) + .with_pr(pr); + // CmdLineRunner::execute blocks, and builds can run for many minutes + let built = tokio::task::block_in_place(|| cmd.execute()); + if let Err(err) = built { + let _ = crate::file::remove_all(&keg); + return Err(err.wrap_err(format!("failed to build {name} {pkg_version} from source"))); + } + if !keg.is_dir() { + bail!( + "build of {name} finished but produced no keg at {}", + keg.display() + ); + } + + let host_tag = tag::host_tag(); + let receipt = pour::write_receipt( + rf, + &host_tag, + &keg, + &Default::default(), + closure, + /* poured_from_bottle */ false, + ); + let linked = receipt.and_then(|()| pour::link_keg(name, &pkg_version, formula.keg_only)); + if let Err(err) = linked { + if let Err(rm_err) = crate::file::remove_all(&keg) { + warn!( + "failed to remove {} after link failure: {rm_err}\n\ + remove it manually, then re-run `mise system install`", + keg.display(), + ); + } + return Err(err); + } + crate::file::remove_all(&build_root)?; + Ok(()) +} + +/// Ensure a mise-managed ruby is installed (precompiled by default) and +/// return the path to its `ruby` executable. +async fn ruby_bin() -> Result { + let mut config = Config::get().await?; + let tool: crate::cli::args::ToolArg = "ruby".parse()?; + let mut ts = ToolsetBuilder::new() + .with_args(&[tool]) + .with_default_to_latest(true) + .build(&config) + .await?; + ts.install_missing_versions( + &mut config, + &InstallOptions { + // only ruby — never drag the rest of the config's toolset in + missing_args_only: true, + reason: "brew source build".to_string(), + ..Default::default() + }, + ) + .await?; + for (backend, tv) in ts.list_current_versions() { + if tv.ba().short != "ruby" { + continue; + } + for bin_dir in backend.list_bin_paths(&config, &tv).await? { + let ruby = bin_dir.join("ruby"); + if ruby.is_file() { + return Ok(ruby); + } + } + } + bail!("failed to provision ruby for building from source (try `mise install ruby`)"); +} + +/// Download the formula's .rb from homebrew/core, pinned to the commit the +/// API metadata was generated from and verified against its sha256. +async fn fetch_formula_rb(formula: &Formula, pr: &dyn SingleReport) -> Result { + // all guaranteed present by check_buildable + let rb_path = formula.ruby_source_path.as_ref().unwrap(); + let sha256 = formula + .ruby_source_checksum + .as_ref() + .and_then(|c| c.sha256.as_deref()) + .unwrap(); + let commit = formula.tap_git_head.as_deref().unwrap(); + let cache_dir = crate::dirs::CACHE.join("system-brew").join("formula"); + let dest = cache_dir.join(format!("{}-{}.rb", formula.name, &sha256[..12])); + if dest.exists() && crate::hash::ensure_checksum(&dest, sha256, None, "sha256").is_ok() { + return Ok(dest); + } + let url = format!("{HOMEBREW_CORE_RAW}/{commit}/{rb_path}"); + pr.set_message(format!("download {rb_path}")); + HTTP_FETCH.download_file(&url, &dest, Some(pr)).await?; + crate::hash::ensure_checksum(&dest, sha256, Some(pr), "sha256")?; + Ok(dest) +} + +/// Download the stable source archive, verified against the API's sha256. +/// the source archive's upstream file name +fn source_basename(formula: &Formula) -> String { + formula + .stable_url() + .map(|src| src.url.as_str()) + .and_then(|url| url.rsplit('/').next()) + .filter(|b| !b.is_empty()) + .unwrap_or("source") + .to_string() +} + +async fn fetch_source(formula: &Formula, pr: &dyn SingleReport) -> Result { + let src = formula.stable_url().unwrap(); // check_buildable + let sha256 = src.checksum.as_deref().unwrap(); // check_buildable + let basename = source_basename(formula); + let cache_dir = crate::dirs::CACHE.join("system-brew").join("sources"); + let dest = cache_dir.join(format!("{}-{basename}", &sha256[..12])); + if dest.exists() && crate::hash::ensure_checksum(&dest, sha256, None, "sha256").is_ok() { + debug!("source cache hit: {}", dest.display()); + return Ok(dest); + } + pr.set_message(format!("download {basename}")); + HTTP_FETCH.download_file(&src.url, &dest, Some(pr)).await?; + crate::hash::ensure_checksum(&dest, sha256, Some(pr), "sha256")?; + Ok(dest) +} + +/// Unpack the source archive the way brew stages it: when the archive holds +/// a single top-level directory, that directory is the buildpath. +fn stage_source(archive: &Path, build_root: &Path, basename: &str) -> Result { + let stage = build_root.join("src"); + crate::file::create_dir_all(&stage)?; + // `basename` is the upstream file name — the cache entry's own name + // carries a checksum prefix that must not leak into the build tree + let format = TarFormat::from_file_name(basename); + if format.is_archive() { + crate::file::extract_archive(archive, &stage, format, &ExtractOptions::default()) + .wrap_err_with(|| format!("failed to extract {}", archive.display()))?; + } else { + // a bare file (script, single binary): stage it as-is + crate::file::copy(archive, stage.join(basename))?; + } + let entries: Vec = crate::file::ls(&stage)?.into_iter().collect(); + match entries.as_slice() { + [single] if single.is_dir() => Ok(single.clone()), + _ => Ok(stage), + } +} + +/// The environment the formula builds in: dependency kegs first on PATH, +/// pkg-config/include/lib flags pointing into the prefix, and the shim's +/// own variables. Mirrors the spirit of brew's superenv without the +/// compiler shims. +fn build_env( + rf: &ResolvedFormula, + closure: &[ResolvedFormula], + pkg_version: &str, + buildpath: &Path, + formula_rb: &Path, +) -> HashMap { + let prefix = prefix::prefix(); + let opt = prefix.join("opt"); + // only this formula's transitive dependencies — unrelated formulae from + // the same install batch must not leak into the build environment + let by_name: HashMap<&str, &ResolvedFormula> = closure + .iter() + .flat_map(|other| { + std::iter::once((other.formula.name.as_str(), other)).chain( + other + .formula + .aliases + .iter() + .map(move |a| (a.as_str(), other)), + ) + }) + .collect(); + // walk each formula's deps under the same variations tag the closure + // resolution used (the dep's selected bottle tag, not the host's) + let host_tag = tag::host_tag(); + let rf_tag = super::resolve::dep_tag(&rf.formula, &host_tag); + let mut deps: Vec<&ResolvedFormula> = vec![]; + let mut seen: std::collections::HashSet<&str> = + std::iter::once(rf.formula.name.as_str()).collect(); + let mut queue: Vec<&String> = rf + .formula + .dependencies_for(&rf_tag) + .iter() + .chain(rf.formula.build_dependencies_for(&rf_tag)) + .collect(); + while let Some(dep) = queue.pop() { + let Some(other) = by_name.get(dep.as_str()) else { + continue; + }; + if !seen.insert(other.formula.name.as_str()) { + continue; + } + deps.push(other); + let other_tag = super::resolve::dep_tag(&other.formula, &host_tag); + queue.extend(other.formula.dependencies_for(&other_tag)); + } + let dep_opts: Vec = deps + .iter() + .map(|other| opt.join(&other.formula.name)) + .filter(|p| p.is_dir()) + .collect(); + + let mut path: Vec = dep_opts + .iter() + .map(|p| p.join("bin")) + .filter(|p| p.is_dir()) + .map(|p| p.display().to_string()) + .collect(); + path.push(prefix.join("bin").display().to_string()); + for dir in ["/usr/local/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin"] { + path.push(dir.to_string()); + } + + let pkg_config_path: Vec = dep_opts + .iter() + .flat_map(|p| [p.join("lib/pkgconfig"), p.join("share/pkgconfig")]) + .chain([prefix.join("lib/pkgconfig"), prefix.join("share/pkgconfig")]) + .filter(|p| p.is_dir()) + .map(|p| p.display().to_string()) + .collect(); + + let mut cppflags: Vec = vec![]; + let mut ldflags: Vec = vec![]; + for dir in dep_opts.iter().chain([&prefix]) { + let include = dir.join("include"); + if include.is_dir() { + cppflags.push(format!("-I{}", include.display())); + } + let lib = dir.join("lib"); + if lib.is_dir() { + ldflags.push(format!("-L{}", lib.display())); + } + } + if cfg!(target_os = "linux") { + // binaries must find brewed libraries at runtime without ldconfig + ldflags.push(format!("-Wl,-rpath,{}", prefix.join("lib").display())); + } + + let jobs = Settings::get().jobs.max(1); + let stable_version = rf.formula.versions.stable.clone().unwrap_or_default(); + let mut env = HashMap::from( + [ + ("MISE_BREW_PREFIX", prefix.display().to_string()), + ("MISE_BREW_CELLAR", prefix::cellar().display().to_string()), + ("MISE_BREW_FORMULA_FILE", formula_rb.display().to_string()), + ("MISE_BREW_NAME", rf.formula.name.clone()), + ("MISE_BREW_VERSION", stable_version), + ("MISE_BREW_PKG_VERSION", pkg_version.to_string()), + ("MISE_BREW_BUILDPATH", buildpath.display().to_string()), + ( + "MISE_BREW_CACHE", + crate::dirs::CACHE + .join("system-brew") + .join("downloads") + .display() + .to_string(), + ), + ("MISE_BREW_MAKE_JOBS", jobs.to_string()), + ("PATH", path.join(":")), + ("MAKEFLAGS", format!("-j{jobs}")), + ("HOMEBREW_PREFIX", prefix.display().to_string()), + ("HOMEBREW_CELLAR", prefix::cellar().display().to_string()), + ( + "CMAKE_PREFIX_PATH", + std::iter::once(prefix.clone()) + .chain(dep_opts.iter().cloned()) + .map(|p| p.display().to_string()) + .collect::>() + .join(":"), + ), + ] + .map(|(k, v)| (k.to_string(), v)), + ); + if !pkg_config_path.is_empty() { + env.insert("PKG_CONFIG_PATH".into(), pkg_config_path.join(":")); + } + if !cppflags.is_empty() { + env.insert("CPPFLAGS".into(), cppflags.join(" ")); + env.insert("CFLAGS".into(), cppflags.join(" ")); + env.insert("CXXFLAGS".into(), cppflags.join(" ")); + } + if !ldflags.is_empty() { + env.insert("LDFLAGS".into(), ldflags.join(" ")); + } + env +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::super::api::{BottleFile, BottleSpec, RubySourceChecksum, SourceUrl, Versions}; + use super::*; + + fn formula(tags: &[&str]) -> Formula { + let files: HashMap = tags + .iter() + .map(|tag| { + ( + tag.to_string(), + BottleFile { + cellar: ":any".to_string(), + url: "https://example.com/bottle.tar.gz".to_string(), + sha256: "0".repeat(64), + }, + ) + }) + .collect(); + let mut bottle = HashMap::new(); + if !tags.is_empty() { + bottle.insert("stable".to_string(), BottleSpec { files }); + } + Formula { + name: "test".to_string(), + aliases: vec![], + versions: Versions { + stable: Some("1.0.0".to_string()), + }, + revision: 0, + keg_only: false, + dependencies: vec![], + build_dependencies: vec![], + bottle, + variations: HashMap::new(), + urls: HashMap::from([( + "stable".to_string(), + SourceUrl { + url: "https://example.com/test-1.0.0.tar.gz".to_string(), + checksum: Some("0".repeat(64)), + using: None, + }, + )]), + ruby_source_path: Some("Formula/t/test.rb".to_string()), + ruby_source_checksum: Some(RubySourceChecksum { + sha256: Some("1".repeat(64)), + }), + tap_git_head: Some("abc123".to_string()), + } + } + + #[test] + fn test_has_bottle() { + // the version-independent "all" tag matches every machine + assert!(has_bottle(&formula(&["all"]))); + assert!(!has_bottle(&formula(&[]))); + } + + #[test] + fn test_missing_bottle_reason() { + assert_eq!( + missing_bottle_reason(&formula(&[])), + "source-only formula, no bottles" + ); + assert_eq!( + missing_bottle_reason(&formula(&["x86_64_linux", "arm64_sonoma"])), + "bottles exist only for: arm64_sonoma, x86_64_linux" + ); + } + + #[test] + fn test_check_buildable() { + assert!(check_buildable(&formula(&[])).is_ok()); + + let mut git_source = formula(&[]); + git_source.urls.get_mut("stable").unwrap().using = Some("git".to_string()); + assert!(check_buildable(&git_source).is_err()); + + let mut no_checksum = formula(&[]); + no_checksum.urls.get_mut("stable").unwrap().checksum = None; + assert!(check_buildable(&no_checksum).is_err()); + + let mut no_url = formula(&[]); + no_url.urls.clear(); + assert!(check_buildable(&no_url).is_err()); + } +} diff --git a/src/system/packages/dnf.rs b/src/system/packages/dnf.rs index 742cb688d4..41944ab457 100644 --- a/src/system/packages/dnf.rs +++ b/src/system/packages/dnf.rs @@ -53,7 +53,7 @@ fn parse_rpm_query(output: &str, requests: &[PackageRequest]) -> Vec &'static str { "dnf" diff --git a/src/system/packages/mod.rs b/src/system/packages/mod.rs index 4c3cdd31c1..142a14b60e 100644 --- a/src/system/packages/mod.rs +++ b/src/system/packages/mod.rs @@ -63,7 +63,11 @@ pub struct InstallOpts { pub update: bool, } -#[async_trait] +// `?Send`: the brew manager's source-build path drives the toolset +// machinery (to provision ruby), which holds non-Send shell state across +// awaits. The driver awaits managers sequentially on one task, so the +// futures never cross threads. +#[async_trait(?Send)] pub trait SystemPackageManager: Send + Sync { /// config key, e.g. "apt", "brew" fn name(&self) -> &'static str; diff --git a/src/system/packages/pacman.rs b/src/system/packages/pacman.rs index 91c99b6477..16cf23206f 100644 --- a/src/system/packages/pacman.rs +++ b/src/system/packages/pacman.rs @@ -73,7 +73,7 @@ fn parse_pacman_query(output: &str, requests: &[PackageRequest]) -> Vec &'static str { "pacman"