From 4cfeee1dc198a340a96ff0acc546d1a8ae867a3c Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 7 May 2021 22:03:32 +0800 Subject: [PATCH] homebrew: support ghcr.io (#93) --- src/homebrew.rs | 162 ++++++++++++++++++++++++++++++++++-------------- src/main.rs | 4 +- src/opts.rs | 2 +- 3 files changed, 119 insertions(+), 49 deletions(-) diff --git a/src/homebrew.rs b/src/homebrew.rs index a043741..f72e536 100644 --- a/src/homebrew.rs +++ b/src/homebrew.rs @@ -2,29 +2,80 @@ //! //! Homebrew source will use brew.sh API to fetch all available bottles. //! It will generate a list of URLs. +//! +//! Reference: https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs +//! MIT License, Copyright (c) 2017 Jian Zeng use crate::common::{Mission, SnapshotConfig, SnapshotPath, TransferURL}; -use crate::error::Result; +use crate::error::{Error, Result}; use crate::timeout::{TryTimeoutExt, TryTimeoutFutureExt}; use crate::traits::{SnapshotStorage, SourceStorage}; +use std::collections::{BTreeMap, HashMap}; use std::time::Duration; use async_trait::async_trait; -use serde_json::Value; +use serde::Deserialize; use slog::info; use structopt::StructOpt; #[derive(Debug, Clone, StructOpt)] -pub struct Homebrew { +pub struct HomebrewConfig { #[structopt(long, default_value = "https://formulae.brew.sh/api/formula.json")] pub api_base: String, - #[structopt(long, default_value = "https://homebrew.bintray.com")] - pub bottles_base: String, #[structopt(long, default_value = "all")] pub arch: String, } +pub struct Homebrew { + pub config: HomebrewConfig, + url_mapping: BTreeMap, +} + +#[derive(Deserialize)] +struct Versions { + stable: Option, + bottle: bool, +} + +#[derive(Deserialize)] +struct Bottle { + stable: Option, +} + +#[derive(Deserialize)] +struct BottleStable { + rebuild: u64, + files: HashMap, +} + +#[derive(Deserialize)] +struct Formula { + name: String, + versions: Versions, + bottle: Bottle, + revision: u64, +} + +#[derive(Deserialize)] +struct Formulae(Vec); + +#[derive(Deserialize)] +struct BottleInfo { + url: String, + #[allow(dead_code)] + sha256: String, +} + +impl Homebrew { + pub fn new(config: HomebrewConfig) -> Self { + Self { + config, + url_mapping: BTreeMap::new(), + } + } +} + #[async_trait] impl SnapshotStorage for Homebrew { async fn snapshot( @@ -40,7 +91,7 @@ impl SnapshotStorage for Homebrew { info!(logger, "fetching API json..."); progress.set_message("fetching API json..."); let data = client - .get(&self.api_base) + .get(&self.config.api_base) .send() .timeout(Duration::from_secs(60)) .await @@ -51,58 +102,75 @@ impl SnapshotStorage for Homebrew { .into_result()?; info!(logger, "parsing..."); - let json: Value = serde_json::from_str(&data).unwrap(); - let packages = json.as_array().unwrap(); - let bottles_base = if self.bottles_base.ends_with('/') { - self.bottles_base.clone() - } else { - format!("{}/", self.bottles_base) - }; - let snapshot: Vec = packages - .iter() - .filter_map(|package| package.as_object()) - .filter_map(|package| { - progress.set_message( - package - .get("name") - .and_then(|name| name.as_str()) - .unwrap_or(""), - ); - package.get("bottle") - }) - .filter_map(|bottles| bottles.as_object()) - .filter_map(|bottles| bottles.get("stable")) - .filter_map(|bottles| bottles.as_object()) - .filter_map(|bottles| bottles.get("files")) - .filter_map(|files| files.as_object()) - .flat_map(|files| files.values()) - .filter_map(|bottle_urls| bottle_urls.get("url")) - .filter_map(|url| url.as_str()) - .filter(|url| self.arch.is_empty() || self.arch == "all" || url.contains(&self.arch)) - .map(|url| url.to_string()) - .map(|url| { - if url.starts_with(&bottles_base) { - url[bottles_base.len()..].to_string() - } else { - panic!("Bottles URL doens't begin with its base: {:?}", url); + let formulae: Formulae = serde_json::from_str(&data).unwrap(); + let mut snapshots = vec![]; + for f in formulae.0 { + progress.set_message(&f.name); + + if f.versions.bottle { + if let Some(versions_stable) = f.versions.stable { + if let Some(bs) = f.bottle.stable { + for (platform, v) in bs.files { + if self.config.arch.is_empty() + || self.config.arch == "all" + || platform == self.config.arch + { + let key = format!( + "{name}-{version}{revision}.{platform}.bottle{rebuild}.tar.gz", + name = f.name, + version = versions_stable, + revision = if f.revision == 0 { + "".to_owned() + } else { + format!("_{}", f.revision) + }, + platform = platform, + rebuild = if bs.rebuild == 0 { + "".to_owned() + } else { + format!(".{}", bs.rebuild) + }, + ); + let key = crate::utils::rewrite_url_string(&gen_map, &key); + self.url_mapping.insert(key.clone(), v.url); + snapshots.push(SnapshotPath::new(key)); + } + } + } } - }) - .map(|url| crate::utils::rewrite_url_string(&gen_map, &url)) - .collect(); + } + } progress.finish_with_message("done"); - Ok(crate::utils::snapshot_string_to_path(snapshot)) + Ok(snapshots) } fn info(&self) -> String { - format!("homebrew, {:?}", self) + format!("homebrew, {:?}", self.config) } } #[async_trait] impl SourceStorage for Homebrew { - async fn get_object(&self, snapshot: &SnapshotPath, _mission: &Mission) -> Result { - Ok(TransferURL(format!("{}/{}", self.bottles_base, snapshot.0))) + async fn get_object(&self, snapshot: &SnapshotPath, mission: &Mission) -> Result { + let url = self + .url_mapping + .get(&snapshot.0) + .expect("no URL for bottle"); + let resp = mission + .client + .get(url) + .header(reqwest::header::AUTHORIZATION, "Bearer QQ==") + .header( + reqwest::header::ACCEPT, + "application/vnd.oci.image.index.v1+json", + ) + .send() + .await?; + if !resp.status().is_success() { + return Err(Error::HTTPError(resp.status())); + } + Ok(TransferURL(resp.url().as_str().to_string())) } } diff --git a/src/main.rs b/src/main.rs index f7d06cd..a765e39 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ use lazy_static::lazy_static; use structopt::StructOpt; use crate::github_release::GitHubRelease; +use crate::homebrew::Homebrew; use common::SnapshotConfig; use error::Result; use file_backend::FileBackend; @@ -137,7 +138,8 @@ fn main() { index_bytes_pipe!(buffer_path, prefix, false, 2) ); } - Source::Homebrew(source) => { + Source::Homebrew(config) => { + let source = Homebrew::new(config); transfer!( opts, source, diff --git a/src/opts.rs b/src/opts.rs index 7320ce0..55a7417 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -5,7 +5,7 @@ use crate::file_backend::FileBackend; use crate::ghcup::Ghcup as GhcupConfig; use crate::github_release::GitHubRelease; use crate::gradle::Gradle; -use crate::homebrew::Homebrew as HomebrewConfig; +use crate::homebrew::HomebrewConfig; use crate::pypi::Pypi as PypiConfig; use crate::rsync::Rsync as RsyncConfig;