Skip to content

Commit

Permalink
new method to fetch ghcup yaml (#98)
Browse files Browse the repository at this point in the history
  • Loading branch information
PhotonQuantum authored Sep 4, 2021
1 parent 9761084 commit acc9e68
Show file tree
Hide file tree
Showing 7 changed files with 271 additions and 99 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ lazy_static = "1.4"
md-5 = "0.9"
rand = "0.8"
regex = "1"
reqwest = {version = "0.11", features = ["stream"]}
reqwest = {version = "0.11", features = ["stream", "json"]}
rusoto_core = "0.46.0"
rusoto_s3 = "0.46.0"
serde = {version = "1.0", features = ["derive"]}
Expand Down
24 changes: 15 additions & 9 deletions src/ghcup/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,8 @@ mod yaml;

#[derive(Debug, Clone, StructOpt)]
pub struct Ghcup {
#[structopt(
long,
help = "Ghcup upstream",
default_value = "https://gitlab.haskell.org/haskell/ghcup-hs/-/raw/master/"
)]
pub ghcup_base: String,
#[structopt(flatten)]
pub ghcup_repo_config: GhcupRepoConfig,
#[structopt(long, default_value = "https://get-ghcup.haskell.org/")]
pub script_url: String,
#[structopt(long, help = "Include legacy versions of packages")]
Expand All @@ -55,6 +51,16 @@ pub struct Ghcup {
pub additional_yaml: CommaSplitVecString,
}

#[derive(Debug, Clone, StructOpt)]
pub struct GhcupRepoConfig {
#[structopt(long, help = "Ghcup gitlab host", default_value = "gitlab.haskell.org")]
host: String,
#[structopt(long, help = "Ghcup gitlab repo", default_value = "haskell/ghcup-hs")]
repo: String,
#[structopt(long, help = "Gitlab fetch pagination", default_value = "100")]
pagination: usize,
}

impl Ghcup {
pub fn get_script(&self) -> GhcupScript {
GhcupScript {
Expand All @@ -63,13 +69,13 @@ impl Ghcup {
}
pub fn get_yaml(&self) -> GhcupYaml {
GhcupYaml {
ghcup_base: self.ghcup_base.clone(),
additional_yaml: self.additional_yaml.clone().into(),
ghcup_repo_config: self.ghcup_repo_config.clone(),
snapmeta_to_remote: Default::default(),
}
}
pub fn get_packages(&self) -> GhcupPackages {
GhcupPackages {
ghcup_base: self.ghcup_base.clone(),
ghcup_repo_config: self.ghcup_repo_config.clone(),
include_old_versions: self.include_old_versions,
}
}
Expand Down
52 changes: 36 additions & 16 deletions src/ghcup/packages.rs
Original file line number Diff line number Diff line change
@@ -1,24 +1,18 @@
use async_trait::async_trait;
use slog::info;
use structopt::StructOpt;
use slog::{info, warn};

use crate::common::{Mission, SnapshotConfig, TransferURL};
use crate::error::Result;
use crate::error::{Error, Result};
use crate::metadata::SnapshotMeta;
use crate::traits::{SnapshotStorage, SourceStorage};

use super::parser::GhcupYamlParser;
use super::utils::get_yaml_url;
use super::parser::{GhcupYamlParser, EXPECTED_CONFIG_VERSION};
use super::utils::{fetch_last_tag, filter_map_file_objs, list_files};
use super::GhcupRepoConfig;

#[derive(Debug, Clone, StructOpt)]
#[derive(Debug, Clone)]
pub struct GhcupPackages {
#[structopt(
long,
help = "Ghcup upstream",
default_value = "https://gitlab.haskell.org/haskell/ghcup-hs/-/raw/master/"
)]
pub ghcup_base: String,
#[structopt(long, help = "Include legacy versions of packages")]
pub ghcup_repo_config: GhcupRepoConfig,
pub include_old_versions: bool,
}

Expand All @@ -32,12 +26,38 @@ impl SnapshotStorage<SnapshotMeta> for GhcupPackages {
let logger = mission.logger;
let progress = mission.progress;
let client = mission.client;

let base_url = self.ghcup_base.trim_end_matches('/');
let repo_config = &self.ghcup_repo_config;

info!(logger, "fetching ghcup config...");
progress.set_message("querying version files");
let latest_yaml_obj = filter_map_file_objs(
list_files(
&client,
repo_config,
fetch_last_tag(&client, repo_config).await?,
)
.await?,
)
.max_by(|x, y| x.version().cmp(&y.version()))
.ok_or_else(|| Error::ProcessError(String::from("no config file found")))?;

if latest_yaml_obj.version() != EXPECTED_CONFIG_VERSION {
warn!(
logger,
"unmatched ghcup config yaml. expected: {}, got: {}",
EXPECTED_CONFIG_VERSION,
latest_yaml_obj.version()
)
}

progress.set_message("downloading version file");
let yaml_url = get_yaml_url(base_url, &client).await?;
let yaml_url = format!(
"https://{}/api/v4/projects/{}/repository/blobs/{}/raw",
repo_config.host,
urlencoding::encode(&repo_config.repo),
latest_yaml_obj.id()
);

progress.set_message("downloading yaml config");
let yaml_data = client.get(&yaml_url).send().await?.bytes().await?;
let ghcup_config: GhcupYamlParser = serde_yaml::from_slice(&yaml_data)?;
Expand Down
10 changes: 7 additions & 3 deletions src/ghcup/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ use std::collections::{HashMap, HashSet};

use serde::Deserialize;

pub(crate) const CONFIG_VERSION: &str = "0.0.7";
use super::utils::Version;

pub const EXPECTED_CONFIG_VERSION: Version = Version::new(0, 0, 6);

#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
Expand Down Expand Up @@ -57,12 +59,14 @@ pub struct Components {
pub ghcup: HashMap<String, Release>,
#[serde(rename = "GHC")]
pub ghc: HashMap<String, Release>,
#[serde(rename = "Stack")]
pub stack: HashMap<String, Release>,
}

impl Components {
pub fn uris(&self, include_old_versions: bool) -> HashSet<&str> {
let fields: [&HashMap<String, Release>; 4] =
[&self.cabal, &self.hls, &self.ghcup, &self.ghc];
let fields: [&HashMap<String, Release>; 5] =
[&self.cabal, &self.hls, &self.ghcup, &self.ghc, &self.stack];
fields
.iter()
.flat_map(|field| {
Expand Down
191 changes: 164 additions & 27 deletions src/ghcup/utils.rs
Original file line number Diff line number Diff line change
@@ -1,34 +1,171 @@
use std::fmt;
use std::fmt::{Debug, Display};
use std::str::FromStr;

use itertools::Itertools;
use lazy_static::lazy_static;
use reqwest::Client;
use serde::{Deserialize, Serialize};

use super::parser::CONFIG_VERSION;
use crate::error::{Error, Result};

pub async fn get_yaml_url<'a>(base_url: &'a str, client: &'a Client) -> Result<String> {
let version_matcher = regex::Regex::new("ghcupURL.*(?P<url>https://.*yaml)").unwrap();

let ghcup_version_module = client
.get(&format!("{}/lib/GHCup/Version.hs", base_url))
.send()
.await?
.text()
.await?;

version_matcher
.captures(ghcup_version_module.as_str())
.and_then(|capture| capture.name("url"))
.map(|group| String::from(group.as_str()))
.ok_or_else(|| {
Error::ProcessError(String::from(
"unable to parse ghcup version from haskell src",
use super::GhcupRepoConfig;

lazy_static! {
static ref YAML_CONFIG_PATTERN: regex::Regex =
regex::Regex::new(r"ghcup-(?P<ver>\d.\d.\d).yaml").unwrap();
}

macro_rules! t {
($e: expr) => {
if let Ok(e) = $e {
e
} else {
return None;
}
};
}

// order is reverted to derive Ord ;)
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub struct Version {
pub patch: usize,
pub minor: usize,
pub major: usize,
}

impl Version {
pub const fn new(major: usize, minor: usize, patch: usize) -> Self {
Self {
major,
minor,
patch,
}
}
}

impl Display for Version {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}.{}.{}", self.major, self.minor, self.patch)
}
}

impl FromStr for Version {
type Err = ();

fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
s.split('.')
.collect_tuple()
.and_then(|(major, minor, patch): (&str, &str, &str)| {
Some(Version::new(
t!(major.parse()),
t!(minor.parse()),
t!(patch.parse()),
))
})
.ok_or(())
}
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileInfo {
id: String,
name: String,
path: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TagInfo {
name: String,
commit: CommitInfo,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommitInfo {
id: String,
}

impl TagInfo {
pub fn id(&self) -> &str {
&self.commit.id
}
}

#[derive(Debug, Clone)]
pub struct ObjectInfo {
id: String,
name: String,
version: Version,
}

impl ObjectInfo {
pub fn id(&self) -> &str {
&self.id
}
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> Version {
self.version
}
}

pub async fn fetch_last_tag(client: &Client, config: &GhcupRepoConfig) -> Result<String> {
let req = client.get(format!(
"https://{}/api/v4/projects/{}/repository/tags",
config.host,
urlencoding::encode(&*config.repo)
));

let tags: Vec<TagInfo> = serde_json::from_slice(&*req.send().await?.bytes().await?)
.map_err(Error::JsonDecodeError)?;

Ok(tags
.first()
.ok_or_else(|| Error::ProcessError(String::from("no tag found")))?
.id()
.to_string())
}

pub async fn list_files(
client: &Client,
config: &GhcupRepoConfig,
commit: String,
) -> Result<Vec<FileInfo>> {
let mut output = Vec::new();
for page in 1.. {
let req = client
.get(format!(
"https://{}/api/v4/projects/{}/repository/tree",
config.host,
urlencoding::encode(&*config.repo)
))
.query(&[("per_page", config.pagination), ("page", page)])
.query(&[("ref", commit.clone())]);
let res: Vec<FileInfo> = serde_json::from_slice(&*req.send().await?.bytes().await?)
.map_err(Error::JsonDecodeError)?;

if !res.is_empty() {
output.extend(res);
} else {
break;
}
}
Ok(output)
}

pub fn filter_map_file_objs(
files: impl IntoIterator<Item = FileInfo>,
) -> impl Iterator<Item = ObjectInfo> {
files.into_iter().filter_map(|f| {
YAML_CONFIG_PATTERN.captures(&*f.name).and_then(|c| {
c.name("ver").and_then(|m| {
Some(ObjectInfo {
id: f.id.clone(),
name: f.name.clone(),
version: t!(Version::from_str(m.as_str())),
})
})
})
.and_then(|url| {
if !url.ends_with(format!("{}.yaml", CONFIG_VERSION).as_str()) {
Err(Error::ProcessError(String::from(
"unsupported version of ghcup config",
)))
} else {
Ok(url)
}
})
})
}
Loading

0 comments on commit acc9e68

Please sign in to comment.