diff --git a/Cargo.lock b/Cargo.lock index 6145358..8dfe295 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -835,7 +835,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790b4292c72618abbab50f787a477014fe15634f96291de45672ce46afe122df" dependencies = [ "atomic", + "parking_lot 0.11.2", "serde", + "tempfile", "toml", "uncased", "version_check", @@ -1657,6 +1659,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.5", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -1664,7 +1677,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.3", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", ] [[package]] @@ -1800,7 +1827,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.1", "protobuf", "thiserror", ] @@ -2320,7 +2347,7 @@ checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "phf_shared", "precomputed-hash", ] @@ -2467,7 +2494,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/Cargo.toml b/Cargo.toml index 8622154..965f31c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,4 +37,5 @@ url = "2.2" [dev-dependencies] tempdir = "0.3" -httpmock = "0.6" \ No newline at end of file +httpmock = "0.6" +figment = { version = "0.10", features = ["test"] } \ No newline at end of file diff --git a/src/common.rs b/src/common.rs index 5ead456..8c8897f 100644 --- a/src/common.rs +++ b/src/common.rs @@ -6,6 +6,8 @@ use std::sync::Arc; use actix_web::body::EitherBody; use actix_web::http::{header, StatusCode}; use actix_web::{HttpRequest, HttpResponse, Responder}; +use figment::providers::{Format, Serialized, Toml}; +use figment::Figment; use percent_encoding::percent_decode; use prometheus::{proto, IntCounter as Counter, IntGauge as Gauge, Opts, Registry}; use reqwest::Client; @@ -157,7 +159,7 @@ pub struct IntelMission { } /// An upstream endpoint override rule. -#[derive(Clone, Deserialize, Debug)] +#[derive(Clone, Deserialize, Debug, Eq, PartialEq)] pub struct EndpointOverride { /// Name of the rule. /// @@ -172,7 +174,7 @@ pub struct EndpointOverride { } /// Endpoints of origin servers. -#[derive(Default, Clone, Deserialize, Debug)] +#[derive(Default, Clone, Deserialize, Debug, Eq, PartialEq)] pub struct Endpoints { pub rust_static: String, pub homebrew_bottles: String, @@ -200,7 +202,7 @@ pub struct Endpoints { } /// Configuration for S3 storage. -#[derive(Default, Clone, Deserialize, Debug)] +#[derive(Default, Clone, Deserialize, Debug, Eq, PartialEq)] pub struct S3Config { /// Name of the S3 storage. pub name: String, @@ -213,7 +215,7 @@ pub struct S3Config { } /// Configuration for Github Release endpoint. -#[derive(Default, Clone, Deserialize, Debug)] +#[derive(Default, Clone, Deserialize, Debug, Eq, PartialEq)] pub struct GithubReleaseConfig { /// Repositories allowed to be cached. /// @@ -222,7 +224,7 @@ pub struct GithubReleaseConfig { } /// Global application config. -#[derive(Default, Clone, Deserialize, Debug)] +#[derive(Default, Clone, Deserialize, Debug, Eq, PartialEq)] pub struct Config { /// Address to listen on. pub address: String, @@ -349,3 +351,96 @@ pub enum IntelObject { /// Cache miss. Origin { task: Task }, } + +pub fn collect_config() -> Config { + let figment = Figment::new() + .merge(Serialized::default("address", "127.0.0.1")) + .merge(Serialized::default("port", 8000)) + .merge(Toml::file("Rocket.toml").nested()) // For backward compatibility + .merge(Toml::file("mirror-intel.toml").nested()); + figment.extract().expect("config") +} + +#[cfg(test)] +mod tests { + use figment::Jail; + + use crate::common::{ + collect_config, EndpointOverride, Endpoints, GithubReleaseConfig, S3Config, + }; + use crate::Config; + + #[test] + fn must_collect_config() { + const MIRROR_INTEL_TOML: &str = include_str!("../tests/config/mirror-intel.toml"); + const ROCKET_TOML: &str = include_str!("../tests/config/Rocket.toml"); + Jail::expect_with(|jail| { + jail.create_file("mirror-intel.toml", MIRROR_INTEL_TOML)?; + jail.create_file("Rocket.toml", ROCKET_TOML)?; + let config = collect_config(); + let expected = Config { + address: "0.0.0.0".into(), // Default values can be overridden. + port: 8000, // Default values takes effect if not overridden. + max_pending_task: 16384, // Values in Rocket.toml must be included. + concurrent_download: 512, // mirror-intel.toml takes precedence. + endpoints: Endpoints { + rust_static: "https://mirrors.tuna.tsinghua.edu.cn/rustup".into(), + homebrew_bottles: "https://homebrew.bintray.com".into(), + pypi_packages: "https://mirrors.bfsu.edu.cn/pypi/web/packages".into(), + fedora_iot: "https://d2ju0wfl996cmc.cloudfront.net".into(), + fedora_ostree: "https://d2uk5hbyrobdzx.cloudfront.net".into(), + flathub: "https://dl.flathub.org/repo".into(), + crates_io: "https://static.crates.io".into(), + dart_pub: "https://mirrors.tuna.tsinghua.edu.cn/dart-pub".into(), + guix: "https://ci.guix.gnu.org".into(), + pytorch_wheels: "https://download.pytorch.org/whl".into(), + linuxbrew_bottles: "https://linuxbrew.bintray.com".into(), + sjtug_internal: "https://github.com/sjtug".into(), + flutter_infra: "https://storage.flutter-io.cn/flutter_infra".into(), + flutter_infra_release: "https://storage.flutter-io.cn/flutter_infra_release" + .into(), + github_release: "https://github.com".into(), + nix_channels_store: "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" + .into(), + pypi_simple: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple".into(), + opam_cache: "https://opam.ocaml.org/cache".into(), + gradle_distribution: "https://services.gradle.org/distributions".into(), + overrides: vec![ + EndpointOverride { + name: "flutter".into(), + pattern: "https://storage.flutter-io.cn/".into(), + replace: "https://storage.googleapis.com/".into(), + }, + EndpointOverride { + name: "tuna".into(), + pattern: "https://mirrors.tuna.tsinghua.edu.cn/".into(), + replace: "https://nanomirrors.tuna.tsinghua.edu.cn/".into(), + }, + ], + s3_only: vec!["voidlinux/".into()], + }, + s3: S3Config { + name: "jCloud S3".into(), + endpoint: "https://s3.jcloud.sjtu.edu.cn".into(), + website_endpoint: "https://s3.jcloud.sjtu.edu.cn".into(), + bucket: "899a892efef34b1b944a19981040f55b-oss01".into(), + }, + user_agent: "mirror-intel / 0.1 (siyuan.internal.sjtug.org)".into(), + file_threshold_mb: 4, + ignore_threshold_mb: 1024, + base_url: "https://mirrors.sjtug.sjtu.edu.cn".into(), + max_retries: 3, + direct_stream_size_kb: 4, + read_only: false, + download_timeout: 3600, + github_release: GithubReleaseConfig { + allow: vec!["sjtug/lug/".into(), "FreeCAD/FreeCAD/".into()], + }, + buffer_path: "/mnt/cache/".into(), + workers: None, + }; + assert_eq!(config, expected); + Ok(()) + }); + } +} diff --git a/src/main.rs b/src/main.rs index d71d6fd..8aeec3a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,8 +11,6 @@ use std::sync::Arc; use actix_web::{guard, web, App, HttpServer}; -use figment::providers::{Format, Toml}; -use figment::Figment; use prometheus::{Encoder, TextEncoder}; use reqwest::{Client, ClientBuilder}; use tokio::sync::mpsc::channel; @@ -101,12 +99,7 @@ async fn main() { LogTracer::init().unwrap(); let _guard = setup_log(); - let figment = Figment::new() - .merge(("address", "127.0.0.1")) - .merge(("port", 8000)) - .merge(Toml::file("Rocket.toml").nested()) // For backward compatibility - .merge(Toml::file("mirror-intel.toml").nested()); - let config: Arc = Arc::new(figment.extract().expect("config")); + let config: Arc = Arc::new(common::collect_config()); info!("checking if bucket is available..."); // check if credentials are set and we have permissions diff --git a/tests/config/Rocket.toml b/tests/config/Rocket.toml new file mode 100644 index 0000000..ac0cde4 --- /dev/null +++ b/tests/config/Rocket.toml @@ -0,0 +1,49 @@ +[default] +address = "0.0.0.0" +concurrent_download = 256 +user_agent = "mirror-intel / 0.1 (siyuan.internal.sjtug.org)" +file_threshold_mb = 4 +ignore_threshold_mb = 1024 +base_url = "https://mirrors.sjtug.sjtu.edu.cn" +ttl = 3 +direct_stream_size_kb = 4 +read_only = false +download_timeout = 3600 +buffer_path = "/mnt/cache/" +# log_level = "off" + +[default.github_release] +allow = ["sjtug/lug/", "FreeCAD/FreeCAD/"] + +[default.endpoints] +rust_static = "https://mirrors.tuna.tsinghua.edu.cn/rustup" +homebrew_bottles = "https://homebrew.bintray.com" +linuxbrew_bottles = "https://linuxbrew.bintray.com" +pypi_packages = "https://mirrors.bfsu.edu.cn/pypi/web/packages" +fedora_iot = "https://d2ju0wfl996cmc.cloudfront.net" +fedora_ostree = "https://d2uk5hbyrobdzx.cloudfront.net" +flathub = "https://dl.flathub.org/repo" +crates_io = "https://static.crates.io" +dart_pub = "https://mirrors.tuna.tsinghua.edu.cn/dart-pub" +guix = "https://ci.guix.gnu.org" +pytorch_wheels = "https://download.pytorch.org/whl" +sjtug_internal = "https://github.com/sjtug" +flutter_infra = "https://storage.flutter-io.cn/flutter_infra" +flutter_infra_release = "https://storage.flutter-io.cn/flutter_infra_release" +github_release = "https://github.com" +nix_channels_store = "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" +overrides = [ + { name = "flutter", pattern = "https://storage.flutter-io.cn/", replace = "https://storage.googleapis.com/" }, + { name = "tuna", pattern = "https://mirrors.tuna.tsinghua.edu.cn/", replace = "https://nanomirrors.tuna.tsinghua.edu.cn/" } +] +s3_only = [ + "voidlinux/" +] +opam_cache = "https://opam.ocaml.org/cache" +pypi_simple = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +gradle_distribution = "https://services.gradle.org/distributions" +[default.s3] +name = "jCloud S3" +endpoint = "https://s3.jcloud.sjtu.edu.cn" +website_endpoint = "https://s3.jcloud.sjtu.edu.cn" +bucket = "899a892efef34b1b944a19981040f55b-oss01" diff --git a/tests/config/mirror-intel.toml b/tests/config/mirror-intel.toml new file mode 100644 index 0000000..d48f014 --- /dev/null +++ b/tests/config/mirror-intel.toml @@ -0,0 +1,4 @@ +[default] +address = "0.0.0.0" +concurrent_download = 512 +max_pending_task = 16384