From adef01f3ddae6956fa19cedf0c9c8af40614afc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Sun, 24 Jul 2022 14:32:23 +1200 Subject: [PATCH] Find best download source out of alternatives (format extension) (#236) --- Cargo.toml | 4 +- src/fetchers.rs | 13 +++-- src/fetchers/gh_crate_meta.rs | 91 ++++++++++++++++++++++------------- src/fetchers/quickinstall.rs | 6 +-- src/formats.rs | 23 ++++++--- src/helpers.rs | 6 +-- src/lib.rs | 2 +- 7 files changed, 93 insertions(+), 52 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e033ca747..e5d048cff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,8 @@ edition = "2021" license = "GPL-3.0" [package.metadata.binstall] -pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ format }" -bin-dir = "{ bin }{ format }" +pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }" +bin-dir = "{ bin }{ binary-ext }" [package.metadata.binstall.overrides.x86_64-pc-windows-msvc] pkg-fmt = "zip" diff --git a/src/fetchers.rs b/src/fetchers.rs index 46917e0fd..166917078 100644 --- a/src/fetchers.rs +++ b/src/fetchers.rs @@ -21,8 +21,15 @@ pub trait Fetcher: Send + Sync { /// Fetch a package and extract async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError>; - /// Check if a package is available for download - async fn check(&self) -> Result; + /// Find the package, if it is available for download + /// + /// This may look for multiple remote targets, but must write (using some form of interior + /// mutability) the best one to the implementing struct in some way so `fetch_and_extract` can + /// proceed without additional work. + /// + /// Must return `true` if a package is available, `false` if none is, and reserve errors to + /// fatal conditions only. + async fn find(&self) -> Result; /// Return the package format fn pkg_fmt(&self) -> PkgFmt; @@ -56,7 +63,7 @@ impl MultiFetcher { pub fn add(&mut self, fetcher: Arc) { self.0.push(( fetcher.clone(), - AutoAbortJoinHandle::new(tokio::spawn(async move { fetcher.check().await })), + AutoAbortJoinHandle::new(tokio::spawn(async move { fetcher.find().await })), )); } diff --git a/src/fetchers/gh_crate_meta.rs b/src/fetchers/gh_crate_meta.rs index f2d96c2e8..7a4381db5 100644 --- a/src/fetchers/gh_crate_meta.rs +++ b/src/fetchers/gh_crate_meta.rs @@ -2,25 +2,21 @@ use std::path::Path; use std::sync::Arc; use log::{debug, info, warn}; +use once_cell::sync::OnceCell; use reqwest::Client; use reqwest::Method; use serde::Serialize; use url::Url; use super::Data; -use crate::{download_and_extract, remote_exists, BinstallError, PkgFmt, Template}; +use crate::{ + download_and_extract, remote_exists, AutoAbortJoinHandle, BinstallError, PkgFmt, Template, +}; pub struct GhCrateMeta { client: Client, data: Data, -} - -impl GhCrateMeta { - fn url(&self) -> Result { - let ctx = Context::from_data(&self.data); - debug!("Using context: {:?}", ctx); - ctx.render_url(&self.data.meta.pkg_url) - } + url: OnceCell, } #[async_trait::async_trait] @@ -29,24 +25,52 @@ impl super::Fetcher for GhCrateMeta { Arc::new(Self { client: client.clone(), data: data.clone(), + url: OnceCell::new(), }) } - async fn check(&self) -> Result { - let url = self.url()?; + async fn find(&self) -> Result { + // build up list of potential URLs + let urls = self.data.meta.pkg_fmt.extensions().iter().map(|ext| { + let ctx = Context::from_data(&self.data, ext); + ctx.render_url(&self.data.meta.pkg_url) + }); + + // go check all potential URLs at once + let checks = urls + .map(|url| { + let client = self.client.clone(); + AutoAbortJoinHandle::new(tokio::spawn(async move { + let url = url?; + info!("Checking for package at: '{url}'"); + remote_exists(client, url.clone(), Method::HEAD) + .await + .map(|exists| (url.clone(), exists)) + })) + }) + .collect::>(); + + // get the first URL that exists + for check in checks { + let (url, exists) = check.await??; + if exists { + if url.scheme() != "https" { + warn!( + "URL is not HTTPS! This may become a hard error in the future, tell the upstream!" + ); + } - if url.scheme() != "https" { - warn!( - "URL is not HTTPS! This may become a hard error in the future, tell the upstream!" - ); + info!("Winning URL is {url}"); + self.url.set(url).unwrap(); // find() is called first + return Ok(true); + } } - info!("Checking for package at: '{url}'"); - remote_exists(&self.client, url, Method::HEAD).await + Ok(false) } async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> { - let url = self.url()?; + let url = self.url.get().unwrap(); // find() is called first info!("Downloading package from: '{url}'"); download_and_extract(&self.client, url, self.pkg_fmt(), dst).await } @@ -56,7 +80,8 @@ impl super::Fetcher for GhCrateMeta { } fn source_name(&self) -> String { - self.url() + self.url + .get() .map(|url| { if let Some(domain) = url.domain() { domain.to_string() @@ -66,7 +91,7 @@ impl super::Fetcher for GhCrateMeta { url.to_string() } }) - .unwrap_or_else(|_| "invalid url template".to_string()) + .unwrap_or_else(|| "invalid url".to_string()) } fn is_third_party(&self) -> bool { @@ -87,11 +112,11 @@ struct Context<'c> { pub version: &'c str, /// Soft-deprecated alias for archive-format - pub format: String, + pub format: &'c str, /// Archive format e.g. tar.gz, zip #[serde(rename = "archive-format")] - pub archive_format: String, + pub archive_format: &'c str, /// Filename extension on the binary, i.e. .exe on Windows, nothing otherwise #[serde(rename = "binary-ext")] @@ -101,15 +126,14 @@ struct Context<'c> { impl<'c> Template for Context<'c> {} impl<'c> Context<'c> { - pub(self) fn from_data(data: &'c Data) -> Self { - let pkg_fmt = data.meta.pkg_fmt.to_string(); + pub(self) fn from_data(data: &'c Data, archive_format: &'c str) -> Self { Self { name: &data.name, repo: data.repo.as_ref().map(|s| &s[..]), target: &data.target, version: &data.version, - format: pkg_fmt.clone(), - archive_format: pkg_fmt, + format: archive_format, + archive_format, binary_ext: if data.target.contains("windows") { ".exe" } else { @@ -119,6 +143,7 @@ impl<'c> Context<'c> { } pub(self) fn render_url(&self, template: &str) -> Result { + debug!("Render {template:?} using context: {:?}", self); Ok(Url::parse(&self.render(template)?)?) } } @@ -144,7 +169,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "tgz"); assert_eq!( ctx.render_url(&data.meta.pkg_url).unwrap(), url("https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz") @@ -163,7 +188,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "tgz"); ctx.render_url(&data.meta.pkg_url).unwrap(); } @@ -182,7 +207,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "tgz"); assert_eq!( ctx.render_url(&data.meta.pkg_url).unwrap(), url("https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz") @@ -206,7 +231,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "tgz"); assert_eq!( ctx.render_url(&data.meta.pkg_url).unwrap(), url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz") @@ -228,7 +253,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "tgz"); assert_eq!( ctx.render_url(&data.meta.pkg_url).unwrap(), url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz") @@ -253,7 +278,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "txz"); assert_eq!( ctx.render_url(&data.meta.pkg_url).unwrap(), url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz") @@ -276,7 +301,7 @@ mod test { meta, }; - let ctx = Context::from_data(&data); + let ctx = Context::from_data(&data, "bin"); assert_eq!( ctx.render_url(&data.meta.pkg_url).unwrap(), url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe") diff --git a/src/fetchers/quickinstall.rs b/src/fetchers/quickinstall.rs index 23c520b95..bdca2762e 100644 --- a/src/fetchers/quickinstall.rs +++ b/src/fetchers/quickinstall.rs @@ -32,17 +32,17 @@ impl super::Fetcher for QuickInstall { }) } - async fn check(&self) -> Result { + async fn find(&self) -> Result { let url = self.package_url(); self.report(); info!("Checking for package at: '{url}'"); - remote_exists(&self.client, Url::parse(&url)?, Method::HEAD).await + remote_exists(self.client.clone(), Url::parse(&url)?, Method::HEAD).await } async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> { let url = self.package_url(); info!("Downloading package from: '{url}'"); - download_and_extract(&self.client, Url::parse(&url)?, self.pkg_fmt(), dst).await + download_and_extract(&self.client, &Url::parse(&url)?, self.pkg_fmt(), dst).await } fn pkg_fmt(&self) -> PkgFmt { diff --git a/src/formats.rs b/src/formats.rs index 1cb1d2ed4..4368638e1 100644 --- a/src/formats.rs +++ b/src/formats.rs @@ -1,11 +1,8 @@ use serde::{Deserialize, Serialize}; -use strum_macros::{Display, EnumString, EnumVariantNames}; +use strum_macros::{Display, EnumString}; /// Binary format enumeration -#[derive( - Debug, Copy, Clone, PartialEq, Serialize, Deserialize, Display, EnumString, EnumVariantNames, -)] -#[strum(serialize_all = "snake_case")] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, EnumString)] #[serde(rename_all = "snake_case")] pub enum PkgFmt { /// Download format is TAR (uncompressed) @@ -31,8 +28,7 @@ impl Default for PkgFmt { } impl PkgFmt { - /// If self is one of the tar based formats, - /// return Some. + /// If self is one of the tar based formats, return Some. pub fn decompose(self) -> PkgFmtDecomposed { match self { PkgFmt::Tar => PkgFmtDecomposed::Tar(TarBasedFmt::Tar), @@ -44,6 +40,19 @@ impl PkgFmt { PkgFmt::Zip => PkgFmtDecomposed::Zip, } } + + /// List of possible file extensions for the format. + pub fn extensions(self) -> &'static [&'static str] { + match self { + PkgFmt::Tar => &["tar"], + PkgFmt::Tbz2 => &["tbz2", "tar.bz2"], + PkgFmt::Tgz => &["tgz", "tar.gz"], + PkgFmt::Txz => &["txz", "tar.xz"], + PkgFmt::Tzstd => &["tzstd", "tzst", "tar.zst"], + PkgFmt::Bin => &["bin", "exe"], + PkgFmt::Zip => &["zip"], + } + } } #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/src/helpers.rs b/src/helpers.rs index 768a37a86..269557fc9 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -113,7 +113,7 @@ pub fn create_reqwest_client( } pub async fn remote_exists( - client: &Client, + client: Client, url: Url, method: Method, ) -> Result { @@ -147,11 +147,11 @@ async fn create_request( /// Download a file from the provided URL and extract it to the provided path. pub async fn download_and_extract>( client: &Client, - url: Url, + url: &Url, fmt: PkgFmt, path: P, ) -> Result<(), BinstallError> { - let stream = create_request(client, url).await?; + let stream = create_request(client, url.clone()).await?; let path = path.as_ref(); debug!("Downloading and extracting to: '{}'", path.display()); diff --git a/src/lib.rs b/src/lib.rs index 8738dc31e..29ccb8f40 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -137,7 +137,7 @@ mod test { assert_eq!( &meta.pkg_url, - "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ format }" + "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }" ); assert_eq!(