feat(deps): support versioning (#116)

* feat(deps): support versioning

* support package.json

Cargo.toml

@@ -99,6 +99,7 @@ pretty_yaml = "0.5"
 yaml_parser = "0.2"
 const_format = "0.2"
 dialoguer = "0.11"
+semver = { version = "1", features = ["serde"] }
 
 [dependencies]
 rari-doc.workspace = true

crates/rari-cli/main.rs

@@ -229,15 +229,6 @@ fn main() -> Result<(), Error> {
         info!("Using env_file: {}", env_file.display())
     }
     let cli = Cli::parse();
-    if !cli.skip_updates {
-        rari_deps::webref_css::update_webref_css(rari_types::globals::data_dir())?;
-        rari_deps::web_features::update_web_features(rari_types::globals::data_dir())?;
-        rari_deps::bcd::update_bcd(rari_types::globals::data_dir())?;
-        rari_deps::mdn_data::update_mdn_data(rari_types::globals::data_dir())?;
-        rari_deps::web_ext_examples::update_web_ext_examples(rari_types::globals::data_dir())?;
-        rari_deps::popularities::update_popularities(rari_types::globals::data_dir())?;
-    }
-
     let fmt_filter =
         filter::Targets::new().with_target("rari_doc", cli.verbose.tracing_level_filter());
 
@@ -249,6 +240,7 @@ fn main() -> Result<(), Error> {
     let cli_filter = filter::Targets::new()
         .with_target("rari", cli_level)
         .with_target("rari_tools", cli_level)
+        .with_target("rari_deps", cli_level)
         .with_target("rari_doc", LevelFilter::OFF);
 
     let memory_filter = filter::Targets::new()
@@ -273,6 +265,15 @@ fn main() -> Result<(), Error> {
         .with(memory_layer.clone().with_filter(memory_filter))
         .init();
 
+    if !cli.skip_updates {
+        rari_deps::webref_css::update_webref_css(rari_types::globals::data_dir())?;
+        rari_deps::web_features::update_web_features(rari_types::globals::data_dir())?;
+        rari_deps::bcd::update_bcd(rari_types::globals::data_dir())?;
+        rari_deps::mdn_data::update_mdn_data(rari_types::globals::data_dir())?;
+        rari_deps::web_ext_examples::update_web_ext_examples(rari_types::globals::data_dir())?;
+        rari_deps::popularities::update_popularities(rari_types::globals::data_dir())?;
+    }
+
     match cli.command {
         Commands::Build(args) => {
             let mut settings = Settings::new()?;

crates/rari-deps/Cargo.toml

@@ -14,6 +14,10 @@ serde_json.workspace = true
 chrono.workspace = true
 thiserror.workspace = true
 reqwest.workspace = true
+url.workspace = true
+indexmap.workspace = true
+tracing.workspace = true
+semver.workspace = true
 
 css-syntax-types = { path = "../css-syntax-types" }
 tar = "0.4"

crates/rari-deps/src/bcd.rs

@@ -2,17 +2,18 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
+use rari_types::globals::deps;
 use rari_utils::io::read_to_string;
 use serde_json::Value;
 
 use crate::error::DepsError;
 use crate::npm::get_package;
 
 pub fn update_bcd(base_path: &Path) -> Result<(), DepsError> {
-    if let Some(path) = get_package("@mdn/browser-compat-data", None, base_path)? {
+    if let Some(path) = get_package("@mdn/browser-compat-data", &deps().bcd, base_path)? {
         extract_spec_urls(&path)?;
     }
-    get_package("web-specs", None, base_path)?;
+    get_package("web-specs", &deps().web_specs, base_path)?;
     Ok(())
 }
 

crates/rari-deps/src/client.rs

@@ -0,0 +1,9 @@
+use reqwest::blocking::Response;
+
+pub fn get(url: impl AsRef<str>) -> reqwest::Result<Response> {
+    reqwest::blocking::ClientBuilder::new()
+        .user_agent("mdn/rari")
+        .build()?
+        .get(url.as_ref())
+        .send()
+}

crates/rari-deps/src/current.rs

@@ -1,8 +1,9 @@
 use chrono::{DateTime, Utc};
+use semver::Version;
 use serde::{Deserialize, Serialize};
 
 #[derive(Deserialize, Serialize, Default, Debug)]
 pub struct Current {
     pub latest_last_check: Option<DateTime<Utc>>,
-    pub version: String,
+    pub current_version: Option<Version>,
 }

crates/rari-deps/src/error.rs

@@ -18,4 +18,6 @@ pub enum DepsError {
     WebRefMissingTarballError,
     #[error("Invalid github version")]
     InvalidGitHubVersion,
+    #[error("Invalid github version")]
+    VersionNotFound,
 }

crates/rari-deps/src/external_json.rs

@@ -6,6 +6,7 @@ use chrono::{DateTime, Duration, Utc};
 use rari_utils::io::read_to_string;
 use serde::{Deserialize, Serialize};
 
+use crate::client::get;
 use crate::error::DepsError;
 
 #[derive(Deserialize, Serialize, Default, Debug)]
@@ -26,7 +27,7 @@ pub fn get_json(name: &str, url: &str, out_path: &Path) -> Result<Option<PathBuf
             fs::remove_dir_all(&package_path)?;
         }
         fs::create_dir_all(&package_path)?;
-        let buf = reqwest::blocking::get(url)?.bytes()?;
+        let buf = get(url)?.bytes()?;
 
         let out_file = package_path.join("data.json");
         let file = File::create(out_file).unwrap();

crates/rari-deps/src/github_release.rs

@@ -4,81 +4,98 @@ use std::path::{Path, PathBuf};
 
 use chrono::{Duration, Utc};
 use rari_utils::io::read_to_string;
-use reqwest::redirect::Policy;
+use semver::{Version, VersionReq};
+use serde::Deserialize;
 
+use crate::client::get;
 use crate::current::Current;
 use crate::error::DepsError;
+#[derive(Deserialize, Debug, Clone)]
+struct VersionEntry {
+    tag_name: Option<String>,
+}
+
+type Releases = Vec<VersionEntry>;
+
+fn get_version(repo: &str, version_req: &VersionReq) -> Result<(Version, String), DepsError> {
+    let releases = get(format!(
+        "https://api.github.com/repos/{repo}/releases?per_page=10"
+    ))?;
 
+    let releases: Releases = serde_json::from_value(releases.json()?)?;
+    if let Some(version) = releases.iter().find_map(|k| {
+        let version = k
+            .tag_name
+            .as_ref()
+            .and_then(|v| Version::parse(v.trim_start_matches('v')).ok());
+        if version
+            .as_ref()
+            .map(|v| version_req.matches(v))
+            .unwrap_or_default()
+        {
+            version.map(|version| (version, k.tag_name.clone().unwrap()))
+        } else {
+            None
+        }
+    }) {
+        Ok(version)
+    } else {
+        Err(DepsError::VersionNotFound)
+    }
+}
 /// Download a github release artifact for a given version (defaults to latest).
 pub fn get_artifact(
-    base_url: &str,
+    repo: &str,
     artifact: &str,
     name: &str,
-    version: Option<&str>,
+    version_req: &Option<VersionReq>,
     out_path: &Path,
 ) -> Result<Option<PathBuf>, DepsError> {
-    let version = version.unwrap_or("latest");
+    let star = VersionReq::default();
+
+    let version_req = version_req.as_ref().unwrap_or(&star);
     let package_path = out_path.join(name);
     let last_check_path = package_path.join("last_check.json");
     let now = Utc::now();
     let current = read_to_string(last_check_path)
         .ok()
         .and_then(|current| serde_json::from_str::<Current>(&current).ok())
         .unwrap_or_default();
-    if version != current.version
-        || version == "latest"
-            && current.latest_last_check.unwrap_or_default() < now - Duration::days(1)
+    if !current
+        .current_version
+        .as_ref()
+        .map(|v| version_req.matches(v))
+        .unwrap_or_default()
+        || current.latest_last_check.unwrap_or_default() < now - Duration::days(1)
     {
-        let prev_url = format!(
-            "{base_url}/releases/download/{}/{artifact}",
-            current.version
-        );
-        let url = if version == "latest" {
-            let client = reqwest::blocking::ClientBuilder::default()
-                .redirect(Policy::none())
-                .build()?;
-            let res = client
-                .get(format!("{base_url}/releases/latest/download/{artifact}"))
-                .send()?;
-            res.headers()
-                .get("location")
-                .ok_or(DepsError::InvalidGitHubVersion)?
-                .to_str()?
-                .to_string()
-        } else {
-            format!("{base_url}/releases/download/{version}/{artifact}")
-        };
+        let (version, tag_name) = get_version(repo, version_req)?;
+        let url = format!("https://github.com/{repo}/releases/download/{tag_name}/{artifact}");
 
         let artifact_path = package_path.join(artifact);
-        let download_update = if artifact_path.exists() {
-            prev_url != url
-        } else {
-            true
-        };
+        let download_update = current.current_version.as_ref() != Some(&version);
 
         if download_update {
+            tracing::info!("Updating {repo} ({artifact}) to {version}");
             if package_path.exists() {
                 fs::remove_dir_all(&package_path)?;
             }
             fs::create_dir_all(&package_path)?;
             let mut buf = vec![];
-            let _ = reqwest::blocking::get(url)?.read_to_end(&mut buf)?;
+            let _ = get(url)?.read_to_end(&mut buf)?;
 
             let file = File::create(artifact_path).unwrap();
             let mut buffed = BufWriter::new(file);
 
             buffed.write_all(&buf)?;
         }
 
-        if version == "latest" {
-            fs::write(
-                package_path.join("last_check.json"),
-                serde_json::to_string_pretty(&Current {
-                    version: version.to_string(),
-                    latest_last_check: Some(now),
-                })?,
-            )?;
-        }
+        fs::write(
+            package_path.join("last_check.json"),
+            serde_json::to_string_pretty(&Current {
+                current_version: Some(version),
+                latest_last_check: Some(now),
+            })?,
+        )?;
         if download_update {
             return Ok(Some(package_path));
         }

crates/rari-deps/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod bcd;
+pub mod client;
 pub mod current;
 pub mod error;
 pub mod external_json;

crates/rari-deps/src/mdn_data.rs

@@ -1,9 +1,11 @@
 use std::path::Path;
 
+use rari_types::globals::deps;
+
 use crate::error::DepsError;
 use crate::npm::get_package;
 
 pub fn update_mdn_data(base_path: &Path) -> Result<(), DepsError> {
-    get_package("mdn-data", None, base_path)?;
+    get_package("mdn-data", &deps().mdn_data, base_path)?;
     Ok(())
 }