repos: add gradle distribution (#44)

src/gradle.rs

@@ -0,0 +1,84 @@
+use crate::common::{Mission, SnapshotConfig};
+use crate::error::Result;
+use crate::timeout::{TryTimeoutExt, TryTimeoutFutureExt};
+use crate::traits::{SnapshotStorage, SourceStorage};
+
+use std::time::Duration;
+
+use async_trait::async_trait;
+use serde_json::Value;
+use slog::info;
+
+#[derive(Debug)]
+pub struct Gradle {
+    pub api_base: String,
+    pub distribution_base: String,
+}
+
+#[async_trait]
+impl SnapshotStorage<String> for Gradle {
+    async fn snapshot(
+        &mut self,
+        mission: Mission,
+        _config: &SnapshotConfig,
+    ) -> Result<Vec<String>> {
+        let logger = mission.logger;
+        let progress = mission.progress;
+        let client = mission.client;
+
+        info!(logger, "fetching API json...");
+        let data = client
+            .get(&self.api_base)
+            .send()
+            .timeout(Duration::from_secs(60))
+            .await
+            .into_result()?
+            .text()
+            .timeout(Duration::from_secs(60))
+            .await
+            .into_result()?;
+
+        info!(logger, "parsing...");
+        let json: Value = serde_json::from_str(&data).unwrap();
+        let packages = json.as_array().unwrap();
+        let snapshot: Vec<String> = packages
+            .into_iter()
+            .filter_map(|package| package.as_object())
+            .filter_map(|package| {
+                progress.set_message(
+                    package
+                        .get("version")
+                        .and_then(|version| version.as_str())
+                        .unwrap_or(""),
+                );
+                if let Some(rc_for) = package.get("rcFor") {
+                    if let Some(rc_for) = rc_for.as_str() {
+                        if rc_for != "" {
+                            return None;
+                        }
+                    }
+                }
+                package.get("downloadUrl")
+            })
+            .filter_map(|url| url.as_str())
+            .filter(|url| url.starts_with(&self.distribution_base))
+            .map(|url| url.to_string())
+            .map(|url| url.replace(&self.distribution_base, ""))
+            .collect();
+
+        progress.finish_with_message("done");
+
+        Ok(snapshot)
+    }
+
+    fn info(&self) -> String {
+        format!("gradle, {:?}", self)
+    }
+}
+
+#[async_trait]
+impl SourceStorage<String, String> for Gradle {
+    async fn get_object(&self, snapshot: String, _mission: &Mission) -> Result<String> {
+        Ok(snapshot)
+    }
+}

src/main.rs

@@ -5,6 +5,7 @@ mod crates_io;
 mod dart;
 mod error;
 mod github_release;
+mod gradle;
 mod homebrew;
 mod html_scanner;
 mod mirror_intel;
@@ -88,6 +89,14 @@ fn main() {
             (@arg version_to_retain: --version_to_retain +takes_value default_value("3") "version to retain")
             (@arg target: --target +takes_value default_value("https://siyuan.internal.sjtug.org/github-release") "mirror-intel target")
         )
+        (@subcommand gradle =>
+            (about: "mirror gradle distribution from brew.sh to siyuan mirror-intel with simple diff transfer")
+            (version: "1.0")
+            (author: "Alex Chi <[email protected]>")
+            (@arg api_base: --api_base +takes_value default_value("https://services.gradle.org/versions/all") "version API")
+            (@arg distribution_base: --distribution_base +takes_value default_value("https://services.gradle.org/distributions/") "distribution base")
+            (@arg target: --target +takes_value default_value("https://siyuan.internal.sjtug.org/gradle/distributions") "mirror-intel target")
+        )
     )
     .get_matches();
 
@@ -264,6 +273,27 @@ fn main() {
                 );
                 transfer.transfer().await.unwrap();
             }
+            ("gradle", Some(sub_matches)) => {
+                let source = gradle::Gradle {
+                    api_base: sub_matches.value_of("api_base").unwrap().to_string(),
+                    distribution_base: sub_matches
+                        .value_of("distribution_base")
+                        .unwrap()
+                        .to_string(),
+                };
+                let target = mirror_intel::MirrorIntel::new(
+                    sub_matches.value_of("target").unwrap().to_string(),
+                );
+                let transfer = simple_diff_transfer::SimpleDiffTransfer::new(
+                    source,
+                    target,
+                    simple_diff_transfer::SimpleDiffTransferConfig {
+                        progress,
+                        snapshot_config,
+                    },
+                );
+                transfer.transfer().await.unwrap();
+            }
             _ => {
                 println!("use ./mirror_clone --help to view commands");
             }