Skip to content

Commit

Permalink
Find best download source out of alternatives (format extension) (#236)
Browse files Browse the repository at this point in the history
  • Loading branch information
@passcod
passcod authored Jul 24, 2022
1 parent 19266a4 commit adef01f
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 52 deletions.
4 changes: 2 additions & 2 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ edition = "2021"
license = "GPL-3.0"

[package.metadata.binstall]
pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ format }"
bin-dir = "{ bin }{ format }"
pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }"
bin-dir = "{ bin }{ binary-ext }"

[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
pkg-fmt = "zip"
Expand Down
13 changes: 10 additions & 3 deletions src/fetchers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,15 @@ pub trait Fetcher: Send + Sync {
/// Fetch a package and extract
async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError>;

/// Check if a package is available for download
async fn check(&self) -> Result<bool, BinstallError>;
/// Find the package, if it is available for download
///
/// This may look for multiple remote targets, but must write (using some form of interior
/// mutability) the best one to the implementing struct in some way so `fetch_and_extract` can
/// proceed without additional work.
///
/// Must return `true` if a package is available, `false` if none is, and reserve errors to
/// fatal conditions only.
async fn find(&self) -> Result<bool, BinstallError>;

/// Return the package format
fn pkg_fmt(&self) -> PkgFmt;
Expand Down Expand Up @@ -56,7 +63,7 @@ impl MultiFetcher {
pub fn add(&mut self, fetcher: Arc<dyn Fetcher>) {
self.0.push((
fetcher.clone(),
AutoAbortJoinHandle::new(tokio::spawn(async move { fetcher.check().await })),
AutoAbortJoinHandle::new(tokio::spawn(async move { fetcher.find().await })),
));
}

Expand Down
91 changes: 58 additions & 33 deletions src/fetchers/gh_crate_meta.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,21 @@ use std::path::Path;
use std::sync::Arc;

use log::{debug, info, warn};
use once_cell::sync::OnceCell;
use reqwest::Client;
use reqwest::Method;
use serde::Serialize;
use url::Url;

use super::Data;
use crate::{download_and_extract, remote_exists, BinstallError, PkgFmt, Template};
use crate::{
download_and_extract, remote_exists, AutoAbortJoinHandle, BinstallError, PkgFmt, Template,
};

pub struct GhCrateMeta {
client: Client,
data: Data,
}

impl GhCrateMeta {
fn url(&self) -> Result<Url, BinstallError> {
let ctx = Context::from_data(&self.data);
debug!("Using context: {:?}", ctx);
ctx.render_url(&self.data.meta.pkg_url)
}
url: OnceCell<Url>,
}

#[async_trait::async_trait]
Expand All @@ -29,24 +25,52 @@ impl super::Fetcher for GhCrateMeta {
Arc::new(Self {
client: client.clone(),
data: data.clone(),
url: OnceCell::new(),
})
}

async fn check(&self) -> Result<bool, BinstallError> {
let url = self.url()?;
async fn find(&self) -> Result<bool, BinstallError> {
// build up list of potential URLs
let urls = self.data.meta.pkg_fmt.extensions().iter().map(|ext| {
let ctx = Context::from_data(&self.data, ext);
ctx.render_url(&self.data.meta.pkg_url)
});

// go check all potential URLs at once
let checks = urls
.map(|url| {
let client = self.client.clone();
AutoAbortJoinHandle::new(tokio::spawn(async move {
let url = url?;
info!("Checking for package at: '{url}'");
remote_exists(client, url.clone(), Method::HEAD)
.await
.map(|exists| (url.clone(), exists))
}))
})
.collect::<Vec<_>>();

// get the first URL that exists
for check in checks {
let (url, exists) = check.await??;
if exists {
if url.scheme() != "https" {
warn!(
"URL is not HTTPS! This may become a hard error in the future, tell the upstream!"
);
}

if url.scheme() != "https" {
warn!(
"URL is not HTTPS! This may become a hard error in the future, tell the upstream!"
);
info!("Winning URL is {url}");
self.url.set(url).unwrap(); // find() is called first
return Ok(true);
}
}

info!("Checking for package at: '{url}'");
remote_exists(&self.client, url, Method::HEAD).await
Ok(false)
}

async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> {
let url = self.url()?;
let url = self.url.get().unwrap(); // find() is called first
info!("Downloading package from: '{url}'");
download_and_extract(&self.client, url, self.pkg_fmt(), dst).await
}
Expand All @@ -56,7 +80,8 @@ impl super::Fetcher for GhCrateMeta {
}

fn source_name(&self) -> String {
self.url()
self.url
.get()
.map(|url| {
if let Some(domain) = url.domain() {
domain.to_string()
Expand All @@ -66,7 +91,7 @@ impl super::Fetcher for GhCrateMeta {
url.to_string()
}
})
.unwrap_or_else(|_| "invalid url template".to_string())
.unwrap_or_else(|| "invalid url".to_string())
}

fn is_third_party(&self) -> bool {
Expand All @@ -87,11 +112,11 @@ struct Context<'c> {
pub version: &'c str,

/// Soft-deprecated alias for archive-format
pub format: String,
pub format: &'c str,

/// Archive format e.g. tar.gz, zip
#[serde(rename = "archive-format")]
pub archive_format: String,
pub archive_format: &'c str,

/// Filename extension on the binary, i.e. .exe on Windows, nothing otherwise
#[serde(rename = "binary-ext")]
Expand All @@ -101,15 +126,14 @@ struct Context<'c> {
impl<'c> Template for Context<'c> {}

impl<'c> Context<'c> {
pub(self) fn from_data(data: &'c Data) -> Self {
let pkg_fmt = data.meta.pkg_fmt.to_string();
pub(self) fn from_data(data: &'c Data, archive_format: &'c str) -> Self {
Self {
name: &data.name,
repo: data.repo.as_ref().map(|s| &s[..]),
target: &data.target,
version: &data.version,
format: pkg_fmt.clone(),
archive_format: pkg_fmt,
format: archive_format,
archive_format,
binary_ext: if data.target.contains("windows") {
".exe"
} else {
Expand All @@ -119,6 +143,7 @@ impl<'c> Context<'c> {
}

pub(self) fn render_url(&self, template: &str) -> Result<Url, BinstallError> {
debug!("Render {template:?} using context: {:?}", self);
Ok(Url::parse(&self.render(template)?)?)
}
}
Expand All @@ -144,7 +169,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "tgz");
assert_eq!(
ctx.render_url(&data.meta.pkg_url).unwrap(),
url("https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz")
Expand All @@ -163,7 +188,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "tgz");
ctx.render_url(&data.meta.pkg_url).unwrap();
}

Expand All @@ -182,7 +207,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "tgz");
assert_eq!(
ctx.render_url(&data.meta.pkg_url).unwrap(),
url("https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz")
Expand All @@ -206,7 +231,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "tgz");
assert_eq!(
ctx.render_url(&data.meta.pkg_url).unwrap(),
url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz")
Expand All @@ -228,7 +253,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "tgz");
assert_eq!(
ctx.render_url(&data.meta.pkg_url).unwrap(),
url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz")
Expand All @@ -253,7 +278,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "txz");
assert_eq!(
ctx.render_url(&data.meta.pkg_url).unwrap(),
url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz")
Expand All @@ -276,7 +301,7 @@ mod test {
meta,
};

let ctx = Context::from_data(&data);
let ctx = Context::from_data(&data, "bin");
assert_eq!(
ctx.render_url(&data.meta.pkg_url).unwrap(),
url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe")
Expand Down
6 changes: 3 additions & 3 deletions src/fetchers/quickinstall.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,17 @@ impl super::Fetcher for QuickInstall {
})
}

async fn check(&self) -> Result<bool, BinstallError> {
async fn find(&self) -> Result<bool, BinstallError> {
let url = self.package_url();
self.report();
info!("Checking for package at: '{url}'");
remote_exists(&self.client, Url::parse(&url)?, Method::HEAD).await
remote_exists(self.client.clone(), Url::parse(&url)?, Method::HEAD).await
}

async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> {
let url = self.package_url();
info!("Downloading package from: '{url}'");
download_and_extract(&self.client, Url::parse(&url)?, self.pkg_fmt(), dst).await
download_and_extract(&self.client, &Url::parse(&url)?, self.pkg_fmt(), dst).await
}

fn pkg_fmt(&self) -> PkgFmt {
Expand Down
23 changes: 16 additions & 7 deletions src/formats.rs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
use serde::{Deserialize, Serialize};
use strum_macros::{Display, EnumString, EnumVariantNames};
use strum_macros::{Display, EnumString};

/// Binary format enumeration
#[derive(
Debug, Copy, Clone, PartialEq, Serialize, Deserialize, Display, EnumString, EnumVariantNames,
)]
#[strum(serialize_all = "snake_case")]
#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, EnumString)]
#[serde(rename_all = "snake_case")]
pub enum PkgFmt {
/// Download format is TAR (uncompressed)
Expand All @@ -31,8 +28,7 @@ impl Default for PkgFmt {
}

impl PkgFmt {
/// If self is one of the tar based formats,
/// return Some.
/// If self is one of the tar based formats, return Some.
pub fn decompose(self) -> PkgFmtDecomposed {
match self {
PkgFmt::Tar => PkgFmtDecomposed::Tar(TarBasedFmt::Tar),
Expand All @@ -44,6 +40,19 @@ impl PkgFmt {
PkgFmt::Zip => PkgFmtDecomposed::Zip,
}
}

/// List of possible file extensions for the format.
pub fn extensions(self) -> &'static [&'static str] {
match self {
PkgFmt::Tar => &["tar"],
PkgFmt::Tbz2 => &["tbz2", "tar.bz2"],
PkgFmt::Tgz => &["tgz", "tar.gz"],
PkgFmt::Txz => &["txz", "tar.xz"],
PkgFmt::Tzstd => &["tzstd", "tzst", "tar.zst"],
PkgFmt::Bin => &["bin", "exe"],
PkgFmt::Zip => &["zip"],
}
}
}

#[derive(Debug, Copy, Clone, PartialEq)]
Expand Down
6 changes: 3 additions & 3 deletions src/helpers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ pub fn create_reqwest_client(
}

pub async fn remote_exists(
client: &Client,
client: Client,
url: Url,
method: Method,
) -> Result<bool, BinstallError> {
Expand Down Expand Up @@ -147,11 +147,11 @@ async fn create_request(
/// Download a file from the provided URL and extract it to the provided path.
pub async fn download_and_extract<P: AsRef<Path>>(
client: &Client,
url: Url,
url: &Url,
fmt: PkgFmt,
path: P,
) -> Result<(), BinstallError> {
let stream = create_request(client, url).await?;
let stream = create_request(client, url.clone()).await?;

let path = path.as_ref();
debug!("Downloading and extracting to: '{}'", path.display());
Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ mod test {

assert_eq!(
&meta.pkg_url,
"{ repo }/releases/download/v{ version }/{ name }-{ target }.{ format }"
"{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }"
);

assert_eq!(
Expand Down

0 comments on commit adef01f

Please sign in to comment.