From 889be7ae6ee100f0c12be5d423d936bd760df3c3 Mon Sep 17 00:00:00 2001 From: LightQuantum Date: Wed, 24 Aug 2022 22:27:16 +0800 Subject: [PATCH 1/6] chore: update dependencies --- Cargo.lock | 918 ++++++++++++++++++++++++++++++----------------------- Cargo.toml | 8 +- 2 files changed, 517 insertions(+), 409 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8c29e6..533ef15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,24 +4,33 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e" +dependencies = [ + "libc", +] + [[package]] name = "arc-swap" -version = "1.2.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73" +checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" dependencies = [ "proc-macro2", "quote", @@ -30,9 +39,9 @@ dependencies = [ [[package]] name = "atomic" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" +checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" dependencies = [ "autocfg", ] @@ -50,15 +59,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base-x" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" [[package]] name = "base64" @@ -74,9 +83,9 @@ checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" @@ -89,21 +98,21 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" [[package]] name = "bytes" -version = "1.0.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "cc" -version = "1.0.67" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cfg-if" @@ -113,23 +122,22 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "libc", + "iana-time-zone", "num-integer", "num-traits", "serde", - "time 0.1.43", "winapi", ] [[package]] name = "const_fn" -version = "0.4.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" [[package]] name = "cookie" @@ -137,15 +145,15 @@ version = "0.15.0-dev" source = "git+https://github.com/SergioBenitez/cookie-rs.git?rev=1c3ca83#1c3ca838543b60a4448d279dc4b903cc7a2bc22a" dependencies = [ "percent-encoding", - "time 0.2.25", + "time 0.2.27", "version_check", ] [[package]] name = "core-foundation" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -153,30 +161,33 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "dc948ebb96241bb40ab73effeb80d9f93afaad49359d159a5e61be51619fe813" +dependencies = [ + "libc", +] [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.0" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if", "crossbeam-utils", @@ -184,20 +195,19 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.3" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ - "autocfg", "cfg-if", - "lazy_static", + "once_cell", ] [[package]] name = "crypto-mac" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ "generic-array", "subtle", @@ -271,24 +281,33 @@ checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" [[package]] name = "either" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "encoding_rs" -version = "0.8.28" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ "cfg-if", ] +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + [[package]] name = "figment" -version = "0.10.3" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c38799b106530aa30f774f7fca6d8f7e5f6234a79f427c4fad3c975eaf678931" +checksum = "790b4292c72618abbab50f787a477014fe15634f96291de45672ce46afe122df" dependencies = [ "atomic", "pear", @@ -337,9 +356,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "futures" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" +checksum = "ab30e97ab6aacfe635fad58f22c2bb06c8b685f7421eb1e064a729e2a5f481fa" dependencies = [ "futures-channel", "futures-core", @@ -352,9 +371,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +checksum = "2bfc52cbddcfd745bf1740338492bb0bd83d76c67b445f91c5fb29fae29ecaa1" dependencies = [ "futures-core", "futures-sink", @@ -362,15 +381,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" +checksum = "d2acedae88d38235936c3922476b10fced7b2b68136f5e3c03c2d5be348a1115" [[package]] name = "futures-executor" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" +checksum = "1d11aa21b5b587a64682c0094c2bdd4df0076c5324961a40cc3abd7f37930528" dependencies = [ "futures-core", "futures-task", @@ -379,17 +398,16 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" +checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5" [[package]] name = "futures-macro" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" +checksum = "0db9cce532b0eae2ccf2766ab246f114b56b9cf6d445e00c2549fbc100ca045d" dependencies = [ - "proc-macro-hack", "proc-macro2", "quote", "syn", @@ -397,21 +415,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" +checksum = "ca0bae1fe9752cf7fd9b0064c674ae63f97b37bc714d745cbde0afb7ec4e6765" [[package]] name = "futures-task" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" +checksum = "842fc63b931f4056a24d59de13fb1272134ce261816e063e634ad0c15cdc5306" [[package]] name = "futures-util" -version = "0.3.13" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +checksum = "f0828a5471e340229c11c77ca80017937ce3c58cb788a17e5f1c2d5c485a9577" dependencies = [ "futures-channel", "futures-core", @@ -422,16 +440,14 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] [[package]] name = "generic-array" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -439,9 +455,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", @@ -456,9 +472,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.1" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" dependencies = [ "bytes", "fnv", @@ -475,15 +491,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.9.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] @@ -496,9 +512,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hmac" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ "crypto-mac", "digest", @@ -506,9 +522,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.3" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", @@ -517,31 +533,32 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.0" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", + "pin-project-lite", ] [[package]] name = "httparse" -version = "1.3.5" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" -version = "0.3.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.4" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ "bytes", "futures-channel", @@ -553,7 +570,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project", + "pin-project-lite", "socket2", "tokio", "tower-service", @@ -574,11 +591,24 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "js-sys", + "wasm-bindgen", + "winapi", +] + [[package]] name = "idna" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -587,9 +617,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", "hashbrown", @@ -597,36 +627,36 @@ dependencies = [ [[package]] name = "inlinable_string" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3094308123a0e9fd59659ce45e22de9f53fc1d2ac6e1feb9fef988e4f76cad77" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "instant" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", ] [[package]] name = "ipnet" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itoa" -version = "0.4.7" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "js-sys" -version = "0.3.48" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" dependencies = [ "wasm-bindgen", ] @@ -639,45 +669,51 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.87" +version = "0.2.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265d751d31d6780a3f956bb5b8022feba2d94eeee5a84ba64f4212eedca42213" +checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" [[package]] name = "lock_api" -version = "0.4.2" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] [[package]] name = "matches" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] -name = "md5" -version = "0.7.0" +name = "md-5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer", + "digest", + "opaque-debug", +] [[package]] name = "memchr" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mime" @@ -687,25 +723,14 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mio" -version = "0.7.9" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" -dependencies = [ - "socket2", - "winapi", + "wasi", + "windows-sys", ] [[package]] @@ -737,9 +762,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.7" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -753,20 +778,11 @@ dependencies = [ "tempfile", ] -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi", -] - [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -774,28 +790,37 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" -version = "1.7.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" [[package]] name = "opaque-debug" @@ -805,29 +830,41 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.32" +version = "0.10.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", "cfg-if", "foreign-types", - "lazy_static", "libc", + "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.60" +version = "0.9.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" dependencies = [ "autocfg", "cc", @@ -838,20 +875,30 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.3", ] [[package]] name = "parking_lot_core" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ "cfg-if", "instant", @@ -861,17 +908,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "paste" -version = "1.0.4" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d65c4d95931acda4498f675e332fcbdc9a06705cd07086c510e9b6009cd1c1" +checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22" [[package]] name = "pear" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ab3a2b792945ed67eadbbdcbd2898f8dd2319392b2a45ac21adea5245cb113" +checksum = "15e44241c5e4c868e3eaa78b7c1848cadd6344ed4f54d029832d32b415a58702" dependencies = [ "inlinable_string", "pear_codegen", @@ -880,9 +940,9 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "620c9c4776ba41b59ab101360c9b1419c0c8c81cd2e6e39fae7109e7425994cb" +checksum = "82a5ca643c2303ecb740d506539deba189e16f2754040a42901cd8105d0282d0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -896,31 +956,11 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "pin-project" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -930,15 +970,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "ppv-lite86" -version = "0.2.10" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro-hack" @@ -946,19 +986,13 @@ version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -976,30 +1010,30 @@ dependencies = [ [[package]] name = "prometheus" -version = "0.11.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39" dependencies = [ "cfg-if", "fnv", "lazy_static", - "parking_lot", + "memchr", + "parking_lot 0.12.1", "protobuf", - "regex", "thiserror", ] [[package]] name = "protobuf" -version = "2.22.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" [[package]] name = "quote" -version = "1.0.9" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ "proc-macro2", ] @@ -1019,24 +1053,23 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core 0.6.2", - "rand_hc", + "rand_core 0.6.3", ] [[package]] name = "rand_chacha" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.2", + "rand_core 0.6.3", ] [[package]] @@ -1056,22 +1089,13 @@ checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" [[package]] name = "rand_core" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" -dependencies = [ - "rand_core 0.6.2", -] - [[package]] name = "rdrand" version = "0.4.0" @@ -1083,37 +1107,38 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", "redox_syscall", + "thiserror", ] [[package]] name = "ref-cast" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300f2a835d808734ee295d45007adacb9ebb29dd3ae2424acfa17930cae541da" +checksum = "ed13bcd201494ab44900a96490291651d200730904221832b9547d24a87d332b" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c38e3aecd2b21cb3959637b883bb3714bc7e43f0268b9a29d3743ee3e55cdd2" +checksum = "5234cd6063258a5e32903b53b1b6ac043a0541c8adc1f610f67b0326c7a578fa" dependencies = [ "proc-macro2", "quote", @@ -1122,21 +1147,20 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remove_dir_all" @@ -1149,15 +1173,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.1" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0460542b551950620a3648c6aa23318ac6b3cd779114bd873209e6e8b5eb1c34" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64", "bytes", "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "hyper", @@ -1171,9 +1196,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "serde", + "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -1196,14 +1224,14 @@ dependencies = [ "log", "memchr", "num_cpus", - "parking_lot", - "rand 0.8.3", + "parking_lot 0.11.2", + "rand 0.8.5", "ref-cast", "rocket_codegen", "rocket_http", "serde", "state", - "time 0.2.25", + "time 0.2.27", "tokio", "ubyte", "version_check", @@ -1234,14 +1262,14 @@ dependencies = [ "indexmap", "log", "mime", - "parking_lot", + "parking_lot 0.11.2", "pear", "percent-encoding", "pin-project-lite", "ref-cast", "smallvec", "state", - "time 0.2.25", + "time 0.2.27", "tokio", "uncased", "unicode-xid", @@ -1250,9 +1278,9 @@ dependencies = [ [[package]] name = "rusoto_core" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02aff20978970d47630f08de5f0d04799497818d16cafee5aec90c4b4d0806cf" +checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" dependencies = [ "async-trait", "base64", @@ -1266,7 +1294,7 @@ dependencies = [ "log", "rusoto_credential", "rusoto_signature", - "rustc_version", + "rustc_version 0.4.0", "serde", "serde_json", "tokio", @@ -1275,9 +1303,9 @@ dependencies = [ [[package]] name = "rusoto_credential" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e91e4c25ea8bfa6247684ff635299015845113baaa93ba8169b9e565701b58e" +checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" dependencies = [ "async-trait", "chrono", @@ -1293,9 +1321,9 @@ dependencies = [ [[package]] name = "rusoto_s3" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc3f56f14ccf91f880b9a9c2d0556d8523e8c155041c54db155b384a1dd1119" +checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d" dependencies = [ "async-trait", "bytes", @@ -1306,26 +1334,27 @@ dependencies = [ [[package]] name = "rusoto_signature" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486e6b1673ab3e0ba1ded284fb444845fe1b7f41d13989a54dd60f62a7b2baa" +checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" dependencies = [ "base64", "bytes", + "chrono", + "digest", "futures", "hex", "hmac", "http", "hyper", "log", - "md5", + "md-5", "percent-encoding", "pin-project-lite", "rusoto_credential", - "rustc_version", + "rustc_version 0.4.0", "serde", "sha2", - "time 0.2.25", "tokio", ] @@ -1335,29 +1364,38 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.13", ] [[package]] name = "rustversion" -version = "1.0.4" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] name = "schannel" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "winapi", + "windows-sys", ] [[package]] @@ -1368,9 +1406,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "2.1.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfd318104249865096c8da1dfabf09ddbb6d0330ea176812a62ec75e40c4166" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" dependencies = [ "bitflags", "core-foundation", @@ -1381,9 +1419,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.1.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" dependencies = [ "core-foundation-sys", "libc", @@ -1398,6 +1436,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711" + [[package]] name = "semver-parser" version = "0.7.0" @@ -1406,18 +1450,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.123" +version = "1.0.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00" dependencies = [ "proc-macro2", "quote", @@ -1426,9 +1470,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" dependencies = [ "itoa", "ryu", @@ -1437,9 +1481,9 @@ dependencies = [ [[package]] name = "serde_urlencoded" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", "itoa", @@ -1449,43 +1493,55 @@ dependencies = [ [[package]] name = "sha1" -version = "0.6.0" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer", "cfg-if", - "cpuid-bool", + "cpufeatures", "digest", "opaque-debug", ] [[package]] name = "shlex" -version = "0.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.2" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] [[package]] name = "slog" @@ -1495,9 +1551,9 @@ checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" [[package]] name = "slog-async" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60813879f820c85dbc4eabf3269befe374591289019775898d56a81a804fbdc" +checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" dependencies = [ "crossbeam-channel", "slog", @@ -1533,9 +1589,9 @@ dependencies = [ [[package]] name = "slog-stdlog" -version = "4.1.0" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8228ab7302adbf4fcb37e66f3cda78003feb521e7fd9e3847ec117a7784d0f5a" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" dependencies = [ "log", "slog", @@ -1544,39 +1600,38 @@ dependencies = [ [[package]] name = "slog-term" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c1e7e5aab61ced6006149ea772770b84a0d16ce0f7885def313e4829946d76" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" dependencies = [ "atty", - "chrono", "slog", "term", "thread_local", + "time 0.3.13", ] [[package]] name = "smallvec" -version = "1.6.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "socket2" -version = "0.3.19" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ - "cfg-if", "libc", "winapi", ] [[package]] name = "standback" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" dependencies = [ "version_check", ] @@ -1594,7 +1649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -1638,19 +1693,19 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" [[package]] name = "subtle" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.60" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -1671,13 +1726,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if", + "fastrand", "libc", - "rand 0.8.3", "redox_syscall", "remove_dir_all", "winapi", @@ -1696,18 +1751,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21" dependencies = [ "proc-macro2", "quote", @@ -1716,36 +1771,38 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" dependencies = [ "once_cell", ] [[package]] name = "time" -version = "0.1.43" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" dependencies = [ + "const_fn", "libc", + "standback", + "stdweb", + "time-macros 0.1.1", + "version_check", "winapi", ] [[package]] name = "time" -version = "0.2.25" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" +checksum = "db76ff9fa4b1458b3c7f077f3ff9887394058460d21e634355b273aaf11eea45" dependencies = [ - "const_fn", + "itoa", "libc", - "standback", - "stdweb", - "time-macros", - "version_check", - "winapi", + "num_threads", + "time-macros 0.2.4", ] [[package]] @@ -1758,11 +1815,17 @@ dependencies = [ "time-macros-impl", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "time-macros-impl" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -1773,9 +1836,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -1788,9 +1851,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.2.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ "autocfg", "bytes", @@ -1799,18 +1862,19 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", + "socket2", "tokio-macros", "winapi", ] [[package]] name = "tokio-macros" -version = "1.1.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ "proc-macro2", "quote", @@ -1829,39 +1893,39 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.3" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", - "log", "pin-project-lite", "tokio", + "tracing", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] [[package]] name = "tower-service" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.25" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ "cfg-if", "pin-project-lite", @@ -1870,11 +1934,11 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.17" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -1885,57 +1949,60 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ubyte" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42756bb9e708855de2f8a98195643dff31a97f0485d90d8467b39dc24be9e8fe" +checksum = "a58e29f263341a29bb79e14ad7fda5f63b1c7e48929bad4c685d7876b1d04e94" dependencies = [ "serde", ] [[package]] name = "uncased" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300932469d646d39929ffe84ad5c1837beecf602519ef5695e485b472de4082b" +checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" dependencies = [ "version_check", ] [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -dependencies = [ - "matches", -] +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "url" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", @@ -1945,15 +2012,15 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" @@ -1967,31 +2034,29 @@ dependencies = [ [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.71" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ "cfg-if", - "serde", - "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.71" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -2000,9 +2065,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.21" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e67a5806118af01f0d9045915676b22aaebecf4178ae7021bc171dab0b897ab" +checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ "cfg-if", "js-sys", @@ -2012,9 +2077,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.71" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2022,9 +2087,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.71" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ "proc-macro2", "quote", @@ -2035,15 +2100,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.71" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" [[package]] name = "web-sys" -version = "0.3.48" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" +checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -2071,29 +2136,72 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + [[package]] name = "winreg" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] [[package]] name = "xml-rs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" [[package]] name = "yansi" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zeroize" -version = "1.2.0" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81a974bcdd357f0dca4d41677db03436324d45a4c9ed2d0b873a5a360ce41c36" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" diff --git a/Cargo.toml b/Cargo.toml index 4fd9dfe..3f49f79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,13 +12,13 @@ futures = "0.3" futures-util = "0.3" lazy_static = "1.4" paste = "1" -prometheus = "0.11" +prometheus = "0.13" regex = "1" reqwest = {version = "0.11", features = ["stream"]} rocket = {git = "https://github.com/SergioBenitez/Rocket", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a"} rocket_codegen = {git = "https://github.com/SergioBenitez/Rocket", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a"} -rusoto_core = "0.46.0" -rusoto_s3 = "0.46.0" +rusoto_core = "0.48.0" +rusoto_s3 = "0.48.0" serde = "1.0" slog = "2.5" slog-async = "2.5" @@ -26,7 +26,7 @@ slog-envlogger = "2.2" slog-term = "2.6" thiserror = "1.0" tokio = {version = "1.0", features = ["full"]} -tokio-util = {version = "0.6", features = ["compat"]} +tokio-util = {version = "0.7", features = ["compat"]} [dev-dependencies] tempdir = "0.3" From ae1c88f6055796195755ca61f7746a1c40b1da8e Mon Sep 17 00:00:00 2001 From: LightQuantum Date: Thu, 25 Aug 2022 03:15:48 +0800 Subject: [PATCH 2/6] chore: better document --- src/artifacts.rs | 89 ++++++++++++++++++++++++++++++++++++------------ src/browse.rs | 18 +++++----- src/common.rs | 76 +++++++++++++++++++++++++++++++++++++++++ src/error.rs | 1 + src/main.rs | 5 +++ src/queue.rs | 1 + src/storage.rs | 12 +++++++ src/utils.rs | 48 +++++++++++++++++++------- 8 files changed, 206 insertions(+), 44 deletions(-) diff --git a/src/artifacts.rs b/src/artifacts.rs index c6de237..fc7d8b9 100644 --- a/src/artifacts.rs +++ b/src/artifacts.rs @@ -1,18 +1,16 @@ -use crate::common::{Config, Metrics, Task}; -use crate::error::{Error, Result}; -use crate::storage::stream_to_s3; - -use futures_util::StreamExt; -use reqwest::Client; -use rocket::http::hyper::Bytes; +//! Artifact download implementation. use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::pin::Pin; +use std::sync::atomic::AtomicUsize; use std::sync::Arc; use futures::{Stream, TryStreamExt}; -use std::sync::atomic::AtomicUsize; +use futures_util::StreamExt; +use reqwest::Client; +use rocket::http::hyper::Bytes; +use slog::{debug, info, o, warn}; use tokio::fs::{self, File, OpenOptions}; use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter}; use tokio::sync::mpsc::{unbounded_channel, Receiver}; @@ -20,14 +18,17 @@ use tokio::sync::Mutex; use tokio::sync::Semaphore; use tokio_util::codec; -use slog::{debug, info, o, warn}; +use crate::common::{Config, Metrics, Task}; +use crate::error::{Error, Result}; +use crate::storage::stream_to_s3; -type IoResult = std::result::Result; +type IOResult = std::result::Result; +// So if we make `Logger` global, we can convert this into a `transpose` on an adhoc trait. fn transform_stream( stream: impl Stream>, logger: slog::Logger, -) -> impl Stream { +) -> impl Stream { stream.map(move |x| { x.map_err(|err| { warn!(logger, "failed to receive data: {:?}", err); @@ -36,11 +37,13 @@ fn transform_stream( }) } +/// Global unique file id counter. static FILE_ID: AtomicUsize = AtomicUsize::new(0); +/// An async file wrapper that can be used as a file-backed stream buffer. struct FileWrapper { path: PathBuf, - pub f: Option>, + f: Option>, } impl AsMut> for FileWrapper { @@ -56,6 +59,9 @@ impl AsRef> for FileWrapper { } impl FileWrapper { + /// Create a new file at the given path. + /// + /// Existing files are truncated. async fn open(path: impl AsRef) -> Result { Ok(Self { path: path.as_ref().to_path_buf(), @@ -71,10 +77,11 @@ impl FileWrapper { }) } + /// Convert this file into a stream of bytes. async fn into_bytes_stream( mut self, logger: slog::Logger, - ) -> Result> { + ) -> Result> { // remove file on disk, but we could still read it let mut f = self.f.take().unwrap(); f.flush().await?; @@ -102,10 +109,14 @@ impl Drop for FileWrapper { } } +/// Convert a stream of bytes to a file-backed one. +/// +/// The old stream is consumed and a new stream with the same contents is returned. +/// It can be used to download large files from the network. async fn to_file_stream( - mut stream: impl Stream + Unpin, + mut stream: impl Stream + Unpin, logger: slog::Logger, -) -> Result> { +) -> Result> { let path = format!( "/mnt/cache/{}", FILE_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) @@ -118,10 +129,14 @@ async fn to_file_stream( Ok(file.into_bytes_stream(logger).await?) } +/// Convert a stream of bytes to a memory-backed one. +/// +/// The old stream is consumed and a new stream with the same contents is returned. +/// It can be used to download small files from the network. async fn to_memory_stream( content_length: usize, - mut stream: impl Stream + Unpin, -) -> Result> { + mut stream: impl Stream + Unpin, +) -> Result> { let mut result = Vec::with_capacity(content_length); while let Some(v) = stream.next().await { let v = v?; @@ -130,6 +145,10 @@ async fn to_memory_stream( Ok(futures::stream::iter(vec![Ok(Bytes::from(result))])) } +/// Cache a task. +/// +/// This function does the actual caching part. +/// It's called in `download_artifact`, which does something like concurrency control and retries. async fn process_task( task: Task, client: Client, @@ -162,17 +181,19 @@ async fn process_task( Ok(()) } +/// Download a stream of bytes from the given url. async fn stream_from_url( client: Client, url: String, config: &Config, logger: slog::Logger, -) -> Result<(u64, Pin + Sync + Send>>)> { +) -> Result<(u64, Pin + Sync + Send>>)> { let response = client.get(&url).send().await?; let status = response.status(); if !status.is_success() { return Err(Error::HTTPError(status)); } + // TODO ehhh I believe this whole thing can be extracted into an adhoc trait, maybe called ".downloaded()"? if let Some(content_length) = response.content_length() { if content_length > config.ignore_threshold_mb * 1024 * 1024 { return Err(Error::TooLarge(())); @@ -201,11 +222,14 @@ async fn stream_from_url( } } +/// Main artifact download task. +/// +/// This function handles concurrency control, queueing, and retries. pub async fn download_artifacts( mut rx: Receiver, client: Client, logger: slog::Logger, - config: &Config, + config: &Config, // TODO what about pass in an Arc, or we generate an Arc at the first place? metrics: Arc, ) { let sem = Arc::new(Semaphore::new(config.concurrent_download)); @@ -213,28 +237,47 @@ pub async fn download_artifacts( let (fail_tx, mut fail_rx) = unbounded_channel(); let config = Arc::new(config.clone()); + // TODO convert to while let Some(_) loop { + // Breaks when either of the channels is closed? + // Should not have much difference with + // `while let Some(_) = select!(fail_rx.recv(), rx.recv())` if nothing goes wrong, + // but it's not clear what happens if a task panics. + // Need to ensure the program terminates when this happens for a quick recovery. let mut task: Task; tokio::select! { val = fail_rx.recv() => { if let Some(val) = val { task = val; } else { break; } } val = rx.recv() => { if let Some(val) = val { task = val; } else { break; } } } + // Apply override rules on the task. task.to_download_task(&config.endpoints.overrides); metrics.task_in_queue.dec(); + // We need to ensure that the total count of pending tasks doesn't exceed the set limit. + // The income `rx` is already bounded by `max_pending_task`, so it's the retried tasks that + // are the problem. + // If a task is retried and current pending queue is full, this will randomly ignore a + // retried task or an incoming task. + + // TODO I don't think the current double queue design is good. We need to prio income over + // retries, i.e. income overtakes retries. if metrics.task_in_queue.get() > config.max_pending_task as i64 { continue; } + // TODO Oh I see why making logger global is blocked. What about replace slog with tracing? let logger = logger.new(o!("storage" => task.storage.clone(), "origin" => task.origin.clone(), "path" => task.path.clone())); if task.ttl == 0 { + // The task has been retries too many times. Skip it. continue; } if config.read_only { + // We are now in read-only mode. Do nothing. + // TODO so what about not running the whole task when in read-only mode? info!(logger, "skipped"); continue; } @@ -242,6 +285,7 @@ pub async fn download_artifacts( let task_hash = task.upstream(); { + // Deduplicate tasks. let mut processing_task = processing_task.lock().await; if processing_task.contains(&task_hash) { info!(logger, "already processing, continue to next task"); @@ -253,7 +297,9 @@ pub async fn download_artifacts( info!(logger, "start download"); metrics.download_counter.inc(); + // Wait for concurrency permit. let permit = Arc::clone(&sem).acquire_owned().await; + let client = client.clone(); let processing_task = processing_task.clone(); let metrics = metrics.clone(); @@ -261,6 +307,7 @@ pub async fn download_artifacts( let config = config.clone(); metrics.task_download.inc(); + // Spawn actual task download task. tokio::spawn(async move { let _permit = permit; let mut task_new = task.clone(); @@ -299,10 +346,10 @@ pub async fn download_artifacts( #[cfg(test)] mod tests { - use super::*; + use slog::Drain; use tempdir::TempDir; - use slog::Drain; + use super::*; fn create_test_logger() -> slog::Logger { let decorator = slog_term::TermDecorator::new().build(); diff --git a/src/browse.rs b/src/browse.rs index 645933a..110bc8e 100644 --- a/src/browse.rs +++ b/src/browse.rs @@ -1,3 +1,5 @@ +//! S3 index page. + use std::io::Cursor; use crate::common::{Config, IntelMission, IntelResponse}; @@ -6,20 +8,15 @@ use crate::{Error, Result}; use rocket::{http::ContentType, response::Redirect, Response, State}; use rusoto_s3::S3; -use serde::Deserialize; - -#[derive(Debug, Deserialize)] -struct Item { - name: String, - source: String, -} +/// Generate a row for given s3 key. fn generate_url(key: &str, last_modified: &str, size: i64, prefix: &str) -> String { let show_key = if key.len() >= prefix.len() { &key[prefix.len()..] } else { key }; + // TODO refactor this if key.ends_with("/") { return format!( r#" @@ -28,19 +25,20 @@ fn generate_url(key: &str, last_modified: &str, size: i64, prefix: &str) -> Stri {} "#, key, show_key, last_modified, size - ); + ) } else { - return format!( + format!( r#" {} {} {} "#, key, show_key, last_modified, size - ); + ) } } +/// Directory index page for a given path. #[get("/?mirror_intel_list")] pub async fn list( path: IntelPath, diff --git a/src/common.rs b/src/common.rs index 3672dcc..87bd65c 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,3 +1,5 @@ +//! Common types. + use prometheus::{IntCounter as Counter, IntGauge as Gauge, Opts, Registry}; use reqwest::Client; use rocket::response; @@ -9,19 +11,27 @@ use crate::{Error, Result}; use std::sync::Arc; +/// A cache task. #[derive(Debug, Clone)] pub struct Task { + /// S3 storage prefix. pub storage: &'static str, + /// Upstream origin. pub origin: String, + /// File path. pub path: String, + /// Max allowed retries. + /// TODO TTL is a bad name and we should change it. pub ttl: usize, } impl Task { + /// Upstream url. pub fn upstream(&self) -> String { format!("{}/{}", self.origin, self.path) } + /// S3 cache. pub fn cached(&self, config: &Config) -> String { format!( "{}/{}/{}/{}", @@ -29,10 +39,12 @@ impl Task { ) } + /// S3 root path. pub fn root_path(&self) -> String { format!("/{}/{}", self.storage, self.path) } + /// S3 key. Percent-encoded path is decoded. pub fn s3_key(&self) -> Result { Ok(format!( "{}/{}", @@ -42,6 +54,8 @@ impl Task { )) } + /// Apply upstream URL override rules on the task. + /// TODO this is a bad name and we should change it. pub fn to_download_task(&mut self, overrides: &[EndpointOverride]) { for endpoint_override in overrides { if self.origin.contains(&endpoint_override.pattern) { @@ -53,12 +67,19 @@ impl Task { } } +/// Prometheus metrics. pub struct Metrics { + /// Count of resolved objects. pub resolve_counter: Counter, + /// Count of objects downloaded from origin. pub download_counter: Counter, + /// Count of objects failed to download. pub failed_download_counter: Counter, + /// Tasks in queue. pub task_in_queue: Gauge, + /// Tasks in progress. pub task_download: Gauge, + // TODO extract this to a separate `gather` method? pub registry: Registry, } @@ -105,21 +126,39 @@ impl Metrics { } } +/// Runtime hub. #[derive(Clone)] pub struct IntelMission { + /// Sender to caching future. + /// + /// You may issue a new caching task by sending it to the caching future. pub tx: Sender, + /// Reqwest client. pub client: Client, + /// Prometheus metrics. pub metrics: Arc, + /// S3 client. + /// + /// This is an anonymous client. pub s3_client: Arc, } +/// An upstream endpoint override rule. #[derive(Clone, Deserialize, Debug)] pub struct EndpointOverride { + /// Name of the rule. + /// + /// Currently this field is only used for a descriptive purpose. pub name: String, + /// Pattern to match against the origin. + /// + /// Note that only plain strings are supported, i.e. no regex, and substring matching is allowed. pub pattern: String, + /// Replacement for the matched pattern. pub replace: String, } +/// Endpoints of origin servers. #[derive(Default, Clone, Deserialize, Debug)] pub struct Endpoints { pub rust_static: String, @@ -141,35 +180,67 @@ pub struct Endpoints { pub pypi_simple: String, pub opam_cache: String, pub gradle_distribution: String, + /// Upstream override rules. pub overrides: Vec, + /// Paths starts with any of these prefixes will be unconditionally redirected to S3 storage. pub s3_only: Vec, } +/// Configuration for S3 storage. #[derive(Default, Clone, Deserialize, Debug)] pub struct S3Config { + /// Endpoint of the S3 service. pub endpoint: String, + /// Bucket name. pub bucket: String, } +/// Configuration for Github Release endpoint. #[derive(Default, Clone, Deserialize, Debug)] pub struct GithubReleaseConfig { + /// Repositories allowed to be cached. + /// + /// Accessing a repository that is not in this list will result in an unconditional redirect. pub allow: Vec, } +/// Global application config. #[derive(Default, Clone, Deserialize, Debug)] pub struct Config { + /// Max pending task allowed. + /// + /// If the count of pending tasks exceeds this value, early tasks will be ignored. pub max_pending_task: usize, + /// Max concurrent download tasks allowed. pub concurrent_download: usize, + /// Upstream endpoints for redirecting, reverse-proxying and caching. pub endpoints: Endpoints, + /// S3 storage to store cached files. pub s3: S3Config, + /// User agent for requests to upstream. pub user_agent: String, + /// Max size of a stream (usually upstream file) to be buffered in memory when processing a task. + /// + /// If the stream is larger than this value, it will be backed by a temporary file. pub file_threshold_mb: u64, + /// Any stream (usually upstream file) larger than this value will be ignored + /// and won't be downloaded when processing a task. pub ignore_threshold_mb: u64, + /// Base URL of this server. pub base_url: String, + /// Max retry times for a failed download. pub ttl: usize, + /// Maximum size of a cached file to be served directly instead of redirect + /// + /// Only works in `dart_pub` currently. pub direct_stream_size_kb: u64, + /// Run in read-only mode. + /// + /// Do not write to S3 storage. pub read_only: bool, + /// Timeout for upstream requests. pub download_timeout: u64, + /// Github release related configs. pub github_release: GithubReleaseConfig, } @@ -192,7 +263,12 @@ macro_rules! impl_from { impl_from! { response::Redirect, IntelResponse<'a>, IntelResponse::Redirect } impl_from! { response::Response<'a>, IntelResponse<'a>, IntelResponse::Response } +/// Intel object to be served to client. +/// +/// Resolve result of a task. pub enum IntelObject { + /// Cache hit. Cached { task: Task, resp: reqwest::Response }, + /// Cache miss. Origin { task: Task }, } diff --git a/src/error.rs b/src/error.rs index e0f3b13..95e8d67 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,3 +1,4 @@ +//! Mirror-intel errors. use std::io::Cursor; use std::result; diff --git a/src/main.rs b/src/main.rs index a06d359..8137415 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,8 @@ use rocket::State; use slog::{o, Drain}; use tokio::sync::mpsc::channel; +/// Create a logger with styled output, env-filter, and async logging. +/// TODO what about a global logger? There's no reason to pass it around. fn create_logger() -> slog::Logger { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse(); @@ -38,6 +40,7 @@ fn create_logger() -> slog::Logger { slog::Logger::root(drain, o!()) } +/// Metrics endpoint. #[get("/metrics")] pub async fn metrics(intel_mission: State<'_, IntelMission>) -> Result> { let mut buffer = vec![]; @@ -65,6 +68,8 @@ async fn rocket() -> rocket::Rocket { info!(logger, "{:?}", config); + // TODO so we are now having a global bounded queue, which will be easily blocked if there're + // too many requests to large files. See issue #24. let (tx, rx) = channel(config.max_pending_task); let client = ClientBuilder::new() .user_agent(&config.user_agent) diff --git a/src/queue.rs b/src/queue.rs index 1ef3802..3ce8821 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -5,6 +5,7 @@ use rocket::{Request, Response}; use crate::common::IntelMission; +/// Middleware that sets `X-Intel-Queue-Length` header to current task queue length. pub struct QueueLength { pub mission: IntelMission, } diff --git a/src/storage.rs b/src/storage.rs index bd32004..c48597a 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,3 +1,4 @@ +//! S3 storage backend. use crate::error::{Error, Result}; use rusoto_core::credential::{AwsCredentials, StaticProvider}; @@ -6,6 +7,7 @@ use rusoto_s3::{S3Client, S3}; use std::time::Duration; use tokio::time::timeout; +/// SJTU JCloud S3 region. fn jcloud_region() -> Region { Region::Custom { name: "jCloud S3".to_string(), @@ -13,10 +15,16 @@ fn jcloud_region() -> Region { } } +/// Creates an authenticated S3 client. +/// +/// The default credential provider is used. fn get_s3_client() -> S3Client { S3Client::new(jcloud_region()) } +/// Creates an anonymous S3 client. +/// +/// It works in read-only mode. pub fn get_anonymous_s3_client() -> S3Client { S3Client::new_with( rusoto_core::request::HttpClient::new().expect("Failed to creat HTTP client"), @@ -25,6 +33,9 @@ pub fn get_anonymous_s3_client() -> S3Client { ) } +/// Takes a stream and save it to S3 storage. +/// +/// This function requires credentials to be present in default credential provider. pub async fn stream_to_s3( path: &str, content_length: u64, @@ -41,6 +52,7 @@ pub async fn stream_to_s3( Ok(s3_client.put_object(req).await?) } +/// Check whether authenticated S3 storage is available. pub async fn check_s3(bucket: &str) -> Result<()> { Ok(timeout(Duration::from_secs(1), async move { let s3_client = get_s3_client(); diff --git a/src/utils.rs b/src/utils.rs index 0ee0d62..e814b91 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -13,6 +13,9 @@ use rocket::{ use tokio_util::compat::FuturesAsyncReadCompatExt; impl Task { + /// Resolve a task. + /// + /// Returns cache if it is available, otherwise schedules a download and returns origin. async fn resolve_internal( self, mission: &IntelMission, @@ -30,6 +33,7 @@ impl Task { return Ok(IntelObject::Cached { task: self, resp }); } else { mission.metrics.task_in_queue.inc(); + // TODO this may block if the queue is full, which is not good. mission .tx .clone() @@ -41,10 +45,26 @@ impl Task { Ok(IntelObject::Origin { task: self }) } + /// Resolve a task. + /// + /// Returns cache if it is available, otherwise schedules a download and returns origin. + /// + /// # Content + /// + /// This method returns content of the cache if it is available. + /// If you don't need it, consider using `resolve_no_content` instead. pub async fn resolve(self, mission: &IntelMission, config: &Config) -> Result { self.resolve_internal(mission, config, false).await } + /// Resolve a task. + /// + /// Returns cache if it is available, otherwise schedules a download and returns origin. + /// + /// # Content + /// + /// This method doesn't return content of the cache if it is available. + /// If you need it, consider using `resolve` instead. pub async fn resolve_no_content( self, mission: &IntelMission, @@ -53,12 +73,14 @@ impl Task { self.resolve_internal(mission, config, true).await } + /// Always resolves to upstream. Neither returns cache nor schedules a download. pub fn resolve_upstream(self) -> IntelObject { IntelObject::Origin { task: self } } } impl IntelObject { + /// Extract original task from the object. pub fn task(&self) -> &Task { match self { IntelObject::Cached { task, .. } => task, @@ -66,6 +88,7 @@ impl IntelObject { } } + /// Get URL target. pub fn target(&self, config: &crate::common::Config) -> String { match self { IntelObject::Cached { task, .. } => task.cached(config), @@ -73,6 +96,7 @@ impl IntelObject { } } + /// Respond with redirection. pub fn redirect(self, config: &crate::common::Config) -> Redirect { match &self { IntelObject::Cached { .. } => Redirect::moved(self.target(config)), @@ -80,10 +104,11 @@ impl IntelObject { } } + /// Respond with rewritten upstream response. pub async fn rewrite_upstream( self, intel_mission: &IntelMission, - below_size_kb: u64, + below_size_kb: u64, // only responses with size below this will be rewritten f: impl Fn(String) -> String, config: &Config, ) -> Result> { @@ -107,6 +132,8 @@ impl IntelObject { Ok(self.redirect(config).into()) } + /// Pass status code from upstream to new response. + /// TODO extract to adhoc trait or method? fn set_status(intel_response: &mut ResponseBuilder, response: &reqwest::Response) { let status = response.status(); // special case for NGINX 499 @@ -119,6 +146,7 @@ impl IntelObject { } } + /// Respond with reverse proxy. pub async fn reverse_proxy( self, intel_mission: &IntelMission, @@ -166,12 +194,14 @@ impl IntelObject { } } + /// Respond with reverse proxy if it's a small cached file, or redirect otherwise. pub async fn stream_small_cached( self, size_kb: u64, intel_mission: &IntelMission, config: &Config, ) -> Result> { + // TODO if let match &self { IntelObject::Cached { resp, .. } => { if let Some(content_length) = resp.content_length() { @@ -187,23 +217,15 @@ impl IntelObject { } } -impl<'a> IntelResponse<'a> { - pub fn content_type(mut self, content_type: ContentType) -> Self { - match &mut self { - IntelResponse::Redirect(_) => {} - IntelResponse::Response(resp) => { - resp.set_header(content_type); - } - } - self - } -} - +/// 404 page. #[catch(404)] pub fn not_found(req: &rocket::Request) -> Response<'static> { no_route_for(&req.uri().to_string()) } +/// No route page. +/// +/// Hint user to redirect to the S3 index page. pub fn no_route_for(mut route: &str) -> Response<'static> { let mut resp = Response::build(); if route.ends_with("/") { From b6a7320aca76eac370953ecfe385478be1b6b25d Mon Sep 17 00:00:00 2001 From: LightQuantum Date: Thu, 25 Aug 2022 11:18:54 +0800 Subject: [PATCH 3/6] feat: do not start download task in read-only mode --- src/artifacts.rs | 7 ------- src/common.rs | 4 +++- src/main.rs | 47 +++++++++++++++++++++++++++-------------------- src/repos.rs | 2 +- src/utils.rs | 17 +++++++++-------- 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/src/artifacts.rs b/src/artifacts.rs index fc7d8b9..12a63cb 100644 --- a/src/artifacts.rs +++ b/src/artifacts.rs @@ -275,13 +275,6 @@ pub async fn download_artifacts( continue; } - if config.read_only { - // We are now in read-only mode. Do nothing. - // TODO so what about not running the whole task when in read-only mode? - info!(logger, "skipped"); - continue; - } - let task_hash = task.upstream(); { diff --git a/src/common.rs b/src/common.rs index 87bd65c..796fcf9 100644 --- a/src/common.rs +++ b/src/common.rs @@ -132,7 +132,9 @@ pub struct IntelMission { /// Sender to caching future. /// /// You may issue a new caching task by sending it to the caching future. - pub tx: Sender, + /// + /// This field can be `None` if mirror-intel is in read-only mode. + pub tx: Option>, /// Reqwest client. pub client: Client, /// Prometheus metrics. diff --git a/src/main.rs b/src/main.rs index 8137415..9408b17 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,9 +68,32 @@ async fn rocket() -> rocket::Rocket { info!(logger, "{:?}", config); - // TODO so we are now having a global bounded queue, which will be easily blocked if there're - // too many requests to large files. See issue #24. - let (tx, rx) = channel(config.max_pending_task); + info!(logger, "starting server..."); + + let metrics = Arc::new(Metrics::new()); + let metrics_download = metrics.clone(); + let tx = (!config.read_only).then(|| { + // TODO so we are now having a global bounded queue, which will be easily blocked if there're + // too many requests to large files. See issue #24. + let (tx, rx) = channel(config.max_pending_task); + + let config_download = config.clone(); + + // Spawn caching future. + tokio::spawn(async move { + download_artifacts( + rx, + Client::new(), + logger, + &config_download, + metrics_download, + ) + .await + }); + + tx + }); + let client = ClientBuilder::new() .user_agent(&config.user_agent) .build() @@ -79,26 +102,10 @@ async fn rocket() -> rocket::Rocket { let mission = IntelMission { tx, client, - metrics: Arc::new(Metrics::new()), + metrics, s3_client: Arc::new(storage::get_anonymous_s3_client()), }; - let config_download = config.clone(); - let metrics_download = mission.metrics.clone(); - - info!(logger, "starting server..."); - - tokio::spawn(async move { - download_artifacts( - rx, - Client::new(), - logger, - &config_download, - metrics_download, - ) - .await - }); - let queue_length_fairing = QueueLength { mission: mission.clone(), }; diff --git a/src/repos.rs b/src/repos.rs index d20a1c1..5940e1a 100644 --- a/src/repos.rs +++ b/src/repos.rs @@ -386,7 +386,7 @@ mod tests { .unwrap(); let mission = IntelMission { - tx, + tx: Some(tx), client, metrics: Arc::new(Metrics::new()), s3_client: Arc::new(get_anonymous_s3_client()), diff --git a/src/utils.rs b/src/utils.rs index e814b91..3d4d2b1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -32,14 +32,15 @@ impl Task { if resp.status().is_success() { return Ok(IntelObject::Cached { task: self, resp }); } else { - mission.metrics.task_in_queue.inc(); - // TODO this may block if the queue is full, which is not good. - mission - .tx - .clone() - .send(self.clone()) - .await - .map_err(|_| Error::SendError(()))?; + if let Some(tx) = &mission.tx { + mission.metrics.task_in_queue.inc(); + // TODO this may block if the queue is full, which is not good. + tx + .clone() + .send(self.clone()) + .await + .map_err(|_| Error::SendError(()))?; + } } } Ok(IntelObject::Origin { task: self }) From 70ce8eb0dd23688d5cf18c053e9eabe11f1bab51 Mon Sep 17 00:00:00 2001 From: LightQuantum Date: Sat, 27 Aug 2022 21:45:47 +0800 Subject: [PATCH 4/6] refactor: cleanup code --- Cargo.lock | 100 +++++++++++++++--------- Cargo.toml | 13 ++-- src/artifacts.rs | 185 ++++++++++++++++++++++++++------------------- src/browse.rs | 64 ++++++++-------- src/common.rs | 72 ++++++++++-------- src/intel_path.rs | 13 ++-- src/intel_query.rs | 10 ++- src/main.rs | 57 ++++++-------- src/repos.rs | 114 ++++++++++++++-------------- src/storage.rs | 27 ++++--- src/utils.rs | 66 ++++++++-------- 11 files changed, 392 insertions(+), 329 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 533ef15..aeb19aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -356,9 +356,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "futures" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab30e97ab6aacfe635fad58f22c2bb06c8b685f7421eb1e064a729e2a5f481fa" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" dependencies = [ "futures-channel", "futures-core", @@ -371,9 +371,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfc52cbddcfd745bf1740338492bb0bd83d76c67b445f91c5fb29fae29ecaa1" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" dependencies = [ "futures-core", "futures-sink", @@ -381,15 +381,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2acedae88d38235936c3922476b10fced7b2b68136f5e3c03c2d5be348a1115" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" [[package]] name = "futures-executor" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d11aa21b5b587a64682c0094c2bdd4df0076c5324961a40cc3abd7f37930528" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" dependencies = [ "futures-core", "futures-task", @@ -398,15 +398,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" [[package]] name = "futures-macro" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0db9cce532b0eae2ccf2766ab246f114b56b9cf6d445e00c2549fbc100ca045d" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" dependencies = [ "proc-macro2", "quote", @@ -415,21 +415,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0bae1fe9752cf7fd9b0064c674ae63f97b37bc714d745cbde0afb7ec4e6765" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" [[package]] name = "futures-task" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "842fc63b931f4056a24d59de13fb1272134ce261816e063e634ad0c15cdc5306" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" [[package]] name = "futures-util" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0828a5471e340229c11c77ca80017937ce3c58cb788a17e5f1c2d5c485a9577" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" dependencies = [ "futures-channel", "futures-core", @@ -544,9 +544,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -593,13 +593,14 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7" dependencies = [ "android_system_properties", "core-foundation-sys", "js-sys", + "once_cell", "wasm-bindgen", "winapi", ] @@ -675,9 +676,9 @@ checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" [[package]] name = "lock_api" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" dependencies = [ "autocfg", "scopeguard", @@ -742,6 +743,7 @@ dependencies = [ "futures-util", "lazy_static", "paste", + "pin-project", "prometheus", "regex", "reqwest", @@ -754,10 +756,12 @@ dependencies = [ "slog-async", "slog-envlogger", "slog-term", + "tap", "tempdir", "thiserror", "tokio", "tokio-util", + "url", ] [[package]] @@ -956,6 +960,26 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1608,7 +1632,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.13", + "time 0.3.14", ] [[package]] @@ -1619,9 +1643,9 @@ checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "10c98bba371b9b22a71a9414e420f92ddeb2369239af08200816169d5e2dd7aa" dependencies = [ "libc", "winapi", @@ -1714,6 +1738,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempdir" version = "0.3.7" @@ -1751,18 +1781,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994" +checksum = "3d0a539a918745651435ac7db7a18761589a94cd7e94cd56999f828bf73c8a57" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21" +checksum = "c251e90f708e16c49a16f4917dc2131e75222b72edfa9cb7f7c58ae56aae0c09" dependencies = [ "proc-macro2", "quote", @@ -1795,9 +1825,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db76ff9fa4b1458b3c7f077f3ff9887394058460d21e634355b273aaf11eea45" +checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" dependencies = [ "itoa", "libc", diff --git a/Cargo.toml b/Cargo.toml index 3f49f79..3e26faf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,11 +12,12 @@ futures = "0.3" futures-util = "0.3" lazy_static = "1.4" paste = "1" +pin-project = "1.0" prometheus = "0.13" regex = "1" -reqwest = {version = "0.11", features = ["stream"]} -rocket = {git = "https://github.com/SergioBenitez/Rocket", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a"} -rocket_codegen = {git = "https://github.com/SergioBenitez/Rocket", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a"} +reqwest = { version = "0.11", features = ["stream"] } +rocket = { git = "https://github.com/SergioBenitez/Rocket", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a" } +rocket_codegen = { git = "https://github.com/SergioBenitez/Rocket", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a" } rusoto_core = "0.48.0" rusoto_s3 = "0.48.0" serde = "1.0" @@ -24,9 +25,11 @@ slog = "2.5" slog-async = "2.5" slog-envlogger = "2.2" slog-term = "2.6" +tap = "1.0" thiserror = "1.0" -tokio = {version = "1.0", features = ["full"]} -tokio-util = {version = "0.7", features = ["compat"]} +tokio = { version = "1.0", features = ["full"] } +tokio-util = { version = "0.7", features = ["compat"] } +url = "2.2" [dev-dependencies] tempdir = "0.3" diff --git a/src/artifacts.rs b/src/artifacts.rs index 12a63cb..937518b 100644 --- a/src/artifacts.rs +++ b/src/artifacts.rs @@ -5,18 +5,23 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::AtomicUsize; use std::sync::Arc; +use std::task::{Context, Poll}; +use bytes::BytesMut; use futures::{Stream, TryStreamExt}; use futures_util::StreamExt; -use reqwest::Client; +use pin_project::pin_project; +use reqwest::{Client, Response}; use rocket::http::hyper::Bytes; -use slog::{debug, info, o, warn}; +use slog::{debug, info, Logger, o, warn}; +use tap::Pipe; use tokio::fs::{self, File, OpenOptions}; use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter}; use tokio::sync::mpsc::{unbounded_channel, Receiver}; use tokio::sync::Mutex; use tokio::sync::Semaphore; use tokio_util::codec; +use url::Url; use crate::common::{Config, Metrics, Task}; use crate::error::{Error, Result}; @@ -24,11 +29,11 @@ use crate::storage::stream_to_s3; type IOResult = std::result::Result; -// So if we make `Logger` global, we can convert this into a `transpose` on an adhoc trait. -fn transform_stream( - stream: impl Stream>, - logger: slog::Logger, -) -> impl Stream { +/// Convert reqwest resp stream to io result stream. +fn into_io_stream( + stream: impl Stream>, + logger: Logger, +) -> impl Stream { stream.map(move |x| { x.map_err(|err| { warn!(logger, "failed to receive data: {:?}", err); @@ -78,10 +83,7 @@ impl FileWrapper { } /// Convert this file into a stream of bytes. - async fn into_bytes_stream( - mut self, - logger: slog::Logger, - ) -> Result> { + async fn into_bytes_stream(mut self, logger: Logger) -> Result> { // remove file on disk, but we could still read it let mut f = self.f.take().unwrap(); f.flush().await?; @@ -95,7 +97,7 @@ impl FileWrapper { f.seek(std::io::SeekFrom::Start(0)).await?; Ok( codec::FramedRead::new(BufReader::new(f), codec::BytesCodec::new()) - .map_ok(|bytes| bytes.freeze()), + .map_ok(BytesMut::freeze), ) } } @@ -113,10 +115,10 @@ impl Drop for FileWrapper { /// /// The old stream is consumed and a new stream with the same contents is returned. /// It can be used to download large files from the network. -async fn to_file_stream( - mut stream: impl Stream + Unpin, - logger: slog::Logger, -) -> Result> { +async fn into_file_stream( + mut stream: impl Stream + Unpin, + logger: Logger, +) -> Result> { let path = format!( "/mnt/cache/{}", FILE_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) @@ -133,10 +135,10 @@ async fn to_file_stream( /// /// The old stream is consumed and a new stream with the same contents is returned. /// It can be used to download small files from the network. -async fn to_memory_stream( +async fn into_memory_stream( content_length: usize, - mut stream: impl Stream + Unpin, -) -> Result> { + mut stream: impl Stream + Unpin, +) -> Result> { let mut result = Vec::with_capacity(content_length); while let Some(v) = stream.next().await { let v = v?; @@ -156,7 +158,7 @@ async fn process_task( logger: slog::Logger, ) -> Result<()> { if client - .head(&task.cached(config)) + .head(task.cached_url(config)) .send() .await? .status() @@ -166,7 +168,7 @@ async fn process_task( return Ok(()); } let (content_length, stream) = - stream_from_url(client, task.upstream(), config, logger.clone()).await?; + stream_from_url(client, task.upstream_url(), config, logger.clone()).await?; info!(logger, "get length={}", content_length); let key = task.s3_key()?; let result = stream_to_s3( @@ -175,83 +177,108 @@ async fn process_task( rusoto_s3::StreamingBody::new(stream), &config.s3.bucket, ) - .await?; + .await?; info!(logger, "upload to bucket"); debug!(logger, "{:?}", result); Ok(()) } -/// Download a stream of bytes from the given url. -async fn stream_from_url( - client: Client, - url: String, - config: &Config, - logger: slog::Logger, -) -> Result<(u64, Pin + Sync + Send>>)> { - let response = client.get(&url).send().await?; - let status = response.status(); - if !status.is_success() { - return Err(Error::HTTPError(status)); +#[pin_project(project = EitherProj)] +enum Either { + Left(#[pin] T), + Right(#[pin] U), +} + +impl Stream for Either + where + T: Stream, + U: Stream, +{ + type Item = O; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.project() { + EitherProj::Left(left) => left.poll_next(cx), + EitherProj::Right(right) => right.poll_next(cx), + } } - // TODO ehhh I believe this whole thing can be extracted into an adhoc trait, maybe called ".downloaded()"? - if let Some(content_length) = response.content_length() { +} + +/// Convert this response into a byte stream. +/// +/// Backing buffer type is chosen based on the content length. +async fn into_stream( + resp: Response, + config: &Config, + logger: Logger, +) -> Result<(u64, impl Stream + Send + Sync)> { + if let Some(content_length) = resp.content_length() { if content_length > config.ignore_threshold_mb * 1024 * 1024 { - return Err(Error::TooLarge(())); - } else if content_length > config.file_threshold_mb * 1024 * 1024 { - info!(logger, "stream mode: file backend"); - let stream = transform_stream(response.bytes_stream(), logger.clone()); - let stream = to_file_stream(stream, logger).await?; - Ok((content_length, Box::pin(stream))) - } else if content_length > 1024 * 1024 { - info!(logger, "stream mode: memory cache"); - let stream = transform_stream(response.bytes_stream(), logger); - let stream = to_memory_stream(content_length as usize, stream).await?; - Ok((content_length, Box::pin(stream))) + Err(Error::TooLarge(())) } else { - info!(logger, "stream mode: direct copy"); - let stream = transform_stream(response.bytes_stream(), logger); - Ok((content_length, Box::pin(stream))) + let io_stream = resp.bytes_stream().pipe(|s| into_io_stream(s, logger.clone())); + if content_length > config.file_threshold_mb * 1024 * 1024 { + info!(logger, "stream mode: file backend"); + let stream = io_stream.pipe(|s| into_file_stream(s, logger)).await?; + Ok((content_length, Either::Left(Either::Left(stream)))) + } else if content_length > 1024 * 1024 { + info!(logger, "stream mode: memory cache"); + let stream = io_stream.pipe(|s| + into_memory_stream(content_length as usize, s)) + .await?; + Ok((content_length, Either::Left(Either::Right(stream)))) + } else { + info!(logger, "stream mode: direct copy"); + Ok((content_length, Either::Right(Either::Left(io_stream)))) + } } } else { info!(logger, "stream mode: direct copy"); - let resp = response.bytes().await?; + let resp = resp.bytes().await?; Ok(( resp.len() as u64, - Box::pin(futures::stream::iter(vec![resp]).map(|x| Ok(x))), + Either::Right(Either::Right(futures::stream::iter(vec![resp]).map(Ok))), )) } } +/// Download a stream of bytes from the given url. +async fn stream_from_url( + client: Client, + url: Url, + config: &Config, + logger: Logger, +) -> Result<(u64, impl Stream + Send + Sync)> { + let response = client.get(url).send().await?; + let status = response.status(); + if !status.is_success() { + return Err(Error::HTTPError(status)); + } + into_stream(response, config, logger).await +} + /// Main artifact download task. /// /// This function handles concurrency control, queueing, and retries. pub async fn download_artifacts( mut rx: Receiver, client: Client, - logger: slog::Logger, - config: &Config, // TODO what about pass in an Arc, or we generate an Arc at the first place? + logger: Logger, + config: Arc, metrics: Arc, ) { let sem = Arc::new(Semaphore::new(config.concurrent_download)); - let processing_task = Arc::new(Mutex::new(HashSet::::new())); + let processing_task = Arc::new(Mutex::new(HashSet::new())); let (fail_tx, mut fail_rx) = unbounded_channel(); - let config = Arc::new(config.clone()); - - // TODO convert to while let Some(_) - loop { - // Breaks when either of the channels is closed? - // Should not have much difference with - // `while let Some(_) = select!(fail_rx.recv(), rx.recv())` if nothing goes wrong, - // but it's not clear what happens if a task panics. - // Need to ensure the program terminates when this happens for a quick recovery. - let mut task: Task; - tokio::select! { - val = fail_rx.recv() => { if let Some(val) = val { task = val; } else { break; } } - val = rx.recv() => { if let Some(val) = val { task = val; } else { break; } } - } + + while let Some(task) = tokio::select! { + val = fail_rx.recv() => val, + val = rx.recv() => val + } { + let mut task: Task = task; // Work around intelliRust bug. // Apply override rules on the task. - task.to_download_task(&config.endpoints.overrides); + task.apply_override(&config.endpoints.overrides); metrics.task_in_queue.dec(); @@ -268,14 +295,14 @@ pub async fn download_artifacts( } // TODO Oh I see why making logger global is blocked. What about replace slog with tracing? - let logger = logger.new(o!("storage" => task.storage.clone(), "origin" => task.origin.clone(), "path" => task.path.clone())); + let logger = logger.new(o!("storage" => task.storage, "origin" => task.origin.clone(), "path" => task.path.clone())); - if task.ttl == 0 { - // The task has been retries too many times. Skip it. + if task.retry_limit == 0 { + // The task has been retried too many times. Skip it. continue; } - let task_hash = task.upstream(); + let task_hash = task.upstream_url(); { // Deduplicate tasks. @@ -291,7 +318,7 @@ pub async fn download_artifacts( metrics.download_counter.inc(); // Wait for concurrency permit. - let permit = Arc::clone(&sem).acquire_owned().await; + let permit = Arc::clone(&sem).acquire_owned().await.unwrap(); let client = client.clone(); let processing_task = processing_task.clone(); @@ -311,9 +338,9 @@ pub async fn download_artifacts( std::time::Duration::from_secs(config.download_timeout), task_fut, ); - if let Err(err) = task_fut.await.unwrap_or_else(|_| Err(Error::Timeout(()))) { - warn!(logger, "{:?}, ttl={}", err, task_new.ttl); - task_new.ttl -= 1; + if let Err(err) = task_fut.await.unwrap_or(Err(Error::Timeout(()))) { + warn!(logger, "{:?}, ttl={}", err, task_new.retry_limit); + task_new.retry_limit -= 1; metrics.failed_download_counter.inc(); { @@ -344,11 +371,11 @@ mod tests { use super::*; - fn create_test_logger() -> slog::Logger { + fn create_test_logger() -> Logger { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse(); let drain = slog_async::Async::new(drain).build().fuse(); - slog::Logger::root(drain, o!()) + Logger::root(drain, o!()) } #[tokio::test] diff --git a/src/browse.rs b/src/browse.rs index 110bc8e..ddf1254 100644 --- a/src/browse.rs +++ b/src/browse.rs @@ -1,14 +1,15 @@ //! S3 index page. +use std::borrow::Cow; use std::io::Cursor; +use rocket::{http::ContentType, response::Redirect, Response, State}; +use rusoto_s3::S3; + use crate::common::{Config, IntelMission, IntelResponse}; use crate::intel_path::IntelPath; use crate::{Error, Result}; -use rocket::{http::ContentType, response::Redirect, Response, State}; -use rusoto_s3::S3; - /// Generate a row for given s3 key. fn generate_url(key: &str, last_modified: &str, size: i64, prefix: &str) -> String { let show_key = if key.len() >= prefix.len() { @@ -16,26 +17,20 @@ fn generate_url(key: &str, last_modified: &str, size: i64, prefix: &str) -> Stri } else { key }; - // TODO refactor this - if key.ends_with("/") { - return format!( - r#" - {} - {} - {} - "#, - key, show_key, last_modified, size - ) + + let href = if key.ends_with('/') { + Cow::Owned(format!("{}?mirror_intel_list", key)) } else { - format!( - r#" + Cow::Borrowed(key) + }; + format!( + r#" {} {} {} "#, - key, show_key, last_modified, size - ) - } + href, show_key, last_modified, size + ) } /// Directory index page for a given path. @@ -51,9 +46,11 @@ pub async fn list( let mirror_clone_list = "mirror_clone_list.html"; // First, check if there is mirror-clone index - let mut req = rusoto_s3::GetObjectRequest::default(); - req.bucket = config.s3.bucket.clone(); - req.key = format!("{}{}", path_slash, mirror_clone_list); + let req = rusoto_s3::GetObjectRequest { + bucket: config.s3.bucket.clone(), + key: format!("{}{}", path_slash, mirror_clone_list), + ..Default::default() + }; let result = tokio::time::timeout( std::time::Duration::from_secs(1), intel_mission.s3_client.get_object(req), @@ -61,16 +58,18 @@ pub async fn list( .await .map_err(|_| Error::Timeout(()))?; - if let Ok(_) = result { + if result.is_ok() { return Ok(Redirect::permanent(format!("{}{}", real_endpoint, mirror_clone_list)).into()); } // Otherwise, generate a dynamic index - let mut req = rusoto_s3::ListObjectsRequest::default(); - req.bucket = config.s3.bucket.clone(); - req.prefix = Some(path_slash.clone()); - req.delimiter = Some("/".to_string()); - req.max_keys = Some(100); + let req = rusoto_s3::ListObjectsRequest { + bucket: config.s3.bucket.clone(), + prefix: Some(path_slash.clone()), + delimiter: Some("/".to_string()), + max_keys: Some(100), + ..Default::default() + }; let result = tokio::time::timeout( std::time::Duration::from_secs(5), intel_mission.s3_client.list_objects(req), @@ -80,12 +79,11 @@ pub async fn list( let mut resp = Response::build(); - let mut body = format!( - r#" + let mut body = r#" .. "# - ); + .to_string(); if let Some(common_prefixes) = result.common_prefixes { let content = common_prefixes @@ -107,9 +105,9 @@ pub async fn list( None } else { x.last_modified.as_ref().and_then(|last_modified| { - x.size.as_ref().and_then(|size| { - Some(generate_url(&key, &last_modified, *size, &path_slash)) - }) + x.size + .as_ref() + .map(|size| generate_url(key, last_modified, *size, &path_slash)) }) } }) diff --git a/src/common.rs b/src/common.rs index 796fcf9..1938c2b 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,16 +1,17 @@ //! Common types. -use prometheus::{IntCounter as Counter, IntGauge as Gauge, Opts, Registry}; +use std::sync::Arc; + +use prometheus::{proto, IntCounter as Counter, IntGauge as Gauge, Opts, Registry}; use reqwest::Client; use rocket::response; use rusoto_s3::S3Client; use serde::Deserialize; use tokio::sync::mpsc::Sender; +use url::Url; use crate::{Error, Result}; -use std::sync::Arc; - /// A cache task. #[derive(Debug, Clone)] pub struct Task { @@ -21,22 +22,22 @@ pub struct Task { /// File path. pub path: String, /// Max allowed retries. - /// TODO TTL is a bad name and we should change it. - pub ttl: usize, + pub retry_limit: usize, } impl Task { /// Upstream url. - pub fn upstream(&self) -> String { - format!("{}/{}", self.origin, self.path) + pub fn upstream_url(&self) -> Url { + Url::parse(&format!("{}/{}", self.origin, self.path)).expect("invalid upstream url") } - /// S3 cache. - pub fn cached(&self, config: &Config) -> String { - format!( + /// S3 cache url. + pub fn cached_url(&self, config: &Config) -> Url { + Url::parse(&format!( "{}/{}/{}/{}", config.s3.endpoint, config.s3.bucket, self.storage, self.path - ) + )) + .expect("invalid cached url") } /// S3 root path. @@ -55,8 +56,7 @@ impl Task { } /// Apply upstream URL override rules on the task. - /// TODO this is a bad name and we should change it. - pub fn to_download_task(&mut self, overrides: &[EndpointOverride]) { + pub fn apply_override(&mut self, overrides: &[EndpointOverride]) { for endpoint_override in overrides { if self.origin.contains(&endpoint_override.pattern) { self.origin = self @@ -79,12 +79,10 @@ pub struct Metrics { pub task_in_queue: Gauge, /// Tasks in progress. pub task_download: Gauge, - // TODO extract this to a separate `gather` method? - pub registry: Registry, } -impl Metrics { - pub fn new() -> Self { +impl Default for Metrics { + fn default() -> Self { let resolve_counter = Counter::with_opts(Opts::new("resolve_counter", "resolved objects")).unwrap(); let download_counter = Counter::with_opts(Opts::new( @@ -101,28 +99,36 @@ impl Metrics { let task_download = Gauge::with_opts(Opts::new("task_download", "tasks processing")).unwrap(); + Self { + resolve_counter, + download_counter, + failed_download_counter, + task_in_queue, + task_download, + } + } +} + +impl Metrics { + pub fn gather(&self) -> Vec { let registry = Registry::new(); registry - .register(Box::new(resolve_counter.clone())) + .register(Box::new(self.resolve_counter.clone())) .unwrap(); registry - .register(Box::new(download_counter.clone())) + .register(Box::new(self.download_counter.clone())) .unwrap(); registry - .register(Box::new(failed_download_counter.clone())) + .register(Box::new(self.failed_download_counter.clone())) .unwrap(); - registry.register(Box::new(task_in_queue.clone())).unwrap(); - registry.register(Box::new(task_download.clone())).unwrap(); - - Self { - registry, - resolve_counter, - download_counter, - task_in_queue, - task_download, - failed_download_counter, - } + registry + .register(Box::new(self.task_in_queue.clone())) + .unwrap(); + registry + .register(Box::new(self.task_download.clone())) + .unwrap(); + registry.gather() } } @@ -231,7 +237,9 @@ pub struct Config { /// Base URL of this server. pub base_url: String, /// Max retry times for a failed download. - pub ttl: usize, + /// It's named as `ttl` for backward compatibility. + #[serde(rename = "ttl")] + pub max_retries: usize, /// Maximum size of a cached file to be served directly instead of redirect /// /// Only works in `dart_pub` currently. diff --git a/src/intel_path.rs b/src/intel_path.rs index 32d12bf..d71760a 100644 --- a/src/intel_path.rs +++ b/src/intel_path.rs @@ -1,6 +1,7 @@ +use std::path::PathBuf; + use rocket::http::uri::{SegmentError, Segments, Uri}; use rocket::request::FromSegments; -use std::path::PathBuf; /// `IntelPath` represents a URL-encoded path which is safe to use both /// on s3 and on a normal filesystem. @@ -10,7 +11,7 @@ pub struct IntelPath(String); impl<'a> FromSegments<'a> for IntelPath { type Error = SegmentError; - fn from_segments(segments: Segments<'a>) -> Result { + fn from_segments(segments: Segments<'a>) -> Result { let mut buf = PathBuf::new(); for segment in segments { @@ -33,7 +34,7 @@ impl<'a> FromSegments<'a> for IntelPath { } else if cfg!(windows) && decoded.contains('\\') { return Err(SegmentError::BadChar('\\')); } else { - buf.push(segment) + buf.push(segment); } } @@ -47,8 +48,8 @@ impl AsRef for IntelPath { } } -impl Into for IntelPath { - fn into(self) -> String { - self.0 +impl From for String { + fn from(path: IntelPath) -> Self { + path.0 } } diff --git a/src/intel_query.rs b/src/intel_query.rs index dcebe94..a6687ca 100644 --- a/src/intel_query.rs +++ b/src/intel_query.rs @@ -1,14 +1,18 @@ +use std::fmt::{Display, Formatter}; + use rocket::request::{FromQuery, Query}; use crate::error::Error; pub struct IntelQuery(String); -impl IntelQuery { - pub fn to_string(self) -> String { - self.0 +impl Display for IntelQuery { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) } +} +impl IntelQuery { pub fn is_empty(&self) -> bool { self.0.is_empty() } diff --git a/src/main.rs b/src/main.rs index 9408b17..9d81232 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,24 +1,3 @@ -mod artifacts; -mod browse; -mod common; -mod error; -mod intel_path; -mod intel_query; -mod queue; -mod repos; -mod storage; -mod utils; - -use artifacts::download_artifacts; -use browse::list; -use common::{Config, IntelMission, Metrics}; -use error::{Error, Result}; -use queue::QueueLength; -use repos::*; -use slog::{info, warn}; -use storage::check_s3; -use utils::not_found; - #[macro_use] extern crate rocket; @@ -27,9 +6,30 @@ use std::sync::Arc; use prometheus::{Encoder, TextEncoder}; use reqwest::{Client, ClientBuilder}; use rocket::State; +use slog::{info, warn}; use slog::{o, Drain}; use tokio::sync::mpsc::channel; +use artifacts::download_artifacts; +use browse::list; +use common::{Config, IntelMission, Metrics}; +use error::{Error, Result}; +use queue::QueueLength; +use repos::*; +use storage::check_s3; +use utils::not_found; + +mod artifacts; +mod browse; +mod common; +mod error; +mod intel_path; +mod intel_query; +mod queue; +mod repos; +mod storage; +mod utils; + /// Create a logger with styled output, env-filter, and async logging. /// TODO what about a global logger? There's no reason to pass it around. fn create_logger() -> slog::Logger { @@ -45,7 +45,7 @@ fn create_logger() -> slog::Logger { pub async fn metrics(intel_mission: State<'_, IntelMission>) -> Result> { let mut buffer = vec![]; let encoder = TextEncoder::new(); - let metric_families = intel_mission.metrics.registry.gather(); + let metric_families = intel_mission.metrics.gather(); encoder .encode(&metric_families, &mut buffer) .map_err(|err| Error::CustomError(format!("failed to encode metrics: {:?}", err)))?; @@ -70,25 +70,18 @@ async fn rocket() -> rocket::Rocket { info!(logger, "starting server..."); - let metrics = Arc::new(Metrics::new()); + let metrics = Arc::new(Metrics::default()); let metrics_download = metrics.clone(); let tx = (!config.read_only).then(|| { // TODO so we are now having a global bounded queue, which will be easily blocked if there're // too many requests to large files. See issue #24. let (tx, rx) = channel(config.max_pending_task); - let config_download = config.clone(); + let config_download = Arc::new(config.clone()); // Spawn caching future. tokio::spawn(async move { - download_artifacts( - rx, - Client::new(), - logger, - &config_download, - metrics_download, - ) - .await + download_artifacts(rx, Client::new(), logger, config_download, metrics_download).await; }); tx diff --git a/src/repos.rs b/src/repos.rs index 5940e1a..6f0ca56 100644 --- a/src/repos.rs +++ b/src/repos.rs @@ -1,3 +1,9 @@ +use lazy_static::lazy_static; +use paste::paste; +use regex::Regex; +use rocket::response::Redirect; +use rocket::State; + use crate::error::Result; use crate::intel_path::IntelPath; use crate::intel_query::IntelQuery; @@ -6,12 +12,6 @@ use crate::{ utils, }; -use lazy_static::lazy_static; -use paste::paste; -use regex::Regex; -use rocket::response::Redirect; -use rocket::State; - macro_rules! simple_intel { ($name:ident, $route:expr, $filter:ident, $proxy:ident) => { paste! { @@ -26,17 +26,17 @@ macro_rules! simple_intel { let path = path.into(); let task = Task { storage: $route, - ttl: config.ttl, + retry_limit: config.max_retries, origin, path, }; if !query.is_empty() { - return Ok(Redirect::found(format!("{}?{}", task.upstream(), query.to_string())).into()); + return Ok(Redirect::found(format!("{}?{}", task.upstream_url(), query.to_string())).into()); } if !$filter(&config, &task.path) { - return Ok(Redirect::moved(task.upstream()).into()); + return Ok(Redirect::moved(task.upstream_url().to_string()).into()); } if $proxy(&task.path) { @@ -66,17 +66,17 @@ macro_rules! simple_intel { let path = path.into(); let task = Task { storage: $route, - ttl: config.ttl, + retry_limit: config.max_retries, origin, path, }; if !query.is_empty() { - return Ok(Redirect::found(format!("{}?{}", task.upstream(), query.to_string())).into()); + return Ok(Redirect::found(format!("{}?{}", task.upstream_url(), query.to_string())).into()); } if !$filter(&config, &task.path) { - return Ok(Redirect::moved(task.upstream()).into()); + return Ok(Redirect::moved(task.upstream_url().to_string()).into()); } if $proxy(&task.path) { @@ -95,11 +95,11 @@ macro_rules! simple_intel { }; } -pub fn disallow_all(_path: &str) -> bool { +pub const fn disallow_all(_path: &str) -> bool { false } -pub fn allow_all(_config: &Config, _path: &str) -> bool { +pub const fn allow_all(_config: &Config, _path: &str) -> bool { true } @@ -222,13 +222,13 @@ pub async fn dart_pub( let path = path.into(); let task = Task { storage: "dart-pub", - ttl: config.ttl, + retry_limit: config.max_retries, origin: origin.clone(), path, }; if !query.is_empty() { - return Ok(Redirect::found(format!("{}?{}", task.upstream(), query.to_string())).into()); + return Ok(Redirect::found(format!("{}?{}", task.upstream_url(), query)).into()); } if task.path.starts_with("api/") { @@ -240,17 +240,15 @@ pub async fn dart_pub( |content| content.replace(&origin, &format!("{}/dart-pub", config.base_url)), &config, ) - .await? - .into()) + .await?) } else if task.path.starts_with("packages/") { Ok(task .resolve(&intel_mission, &config) .await? .stream_small_cached(config.direct_stream_size_kb, &intel_mission, &config) - .await? - .into()) + .await?) } else { - Ok(Redirect::moved(task.upstream()).into()) + Ok(Redirect::moved(task.upstream_url().to_string()).into()) } } @@ -265,17 +263,16 @@ pub async fn pypi( let path = path.into(); let task = Task { storage: "pypi", - ttl: config.ttl, + retry_limit: config.max_retries, origin: origin.clone(), path, }; if !query.is_empty() { - return Ok(Redirect::found(format!("{}?{}", task.upstream(), query.to_string())).into()); + return Ok(Redirect::found(format!("{}?{}", task.upstream_url(), query)).into()); } - Ok(task - .resolve_upstream() + task.resolve_upstream() .rewrite_upstream( &intel_mission, 4096, @@ -287,8 +284,7 @@ pub async fn pypi( }, &config, ) - .await? - .into()) + .await } macro_rules! nix_intel { @@ -305,13 +301,13 @@ macro_rules! nix_intel { let path = path.into(); let task = Task { storage: $route, - ttl: config.ttl, + retry_limit: config.max_retries, origin, path, }; if !query.is_empty() { - return Ok(Redirect::found(format!("{}?{}", task.upstream(), query.to_string())).into()); + return Ok(Redirect::found(format!("{}?{}", task.upstream_url(), query.to_string())).into()); } if task.path.starts_with("nar/") { @@ -329,7 +325,7 @@ macro_rules! nix_intel { .await? .into()) } else { - Ok(Redirect::moved(task.upstream()).into()) + Ok(Redirect::moved(task.upstream_url().to_string()).into()) } } } @@ -359,16 +355,18 @@ pub async fn index(path: IntelPath, config: State<'_, Config>) -> IntelResponse< #[cfg(test)] mod tests { + use std::sync::Arc; + + use reqwest::ClientBuilder; + use rocket::http::Status; + use tokio::sync::mpsc::{channel, Receiver}; + use crate::queue::QueueLength; use crate::utils::not_found; use crate::{ common::{Config, EndpointOverride, IntelMission, Metrics}, storage::get_anonymous_s3_client, }; - use reqwest::ClientBuilder; - use rocket::http::Status; - use std::sync::Arc; - use tokio::sync::mpsc::{channel, Receiver}; use super::*; @@ -388,7 +386,7 @@ mod tests { let mission = IntelMission { tx: Some(tx), client, - metrics: Arc::new(Metrics::new()), + metrics: Arc::new(Metrics::default()), s3_client: Arc::new(get_anonymous_s3_client()), }; @@ -425,7 +423,7 @@ mod tests { storage: "sjtug-internal", origin: "https://github.com/sjtug".to_string(), path: "mirror-clone/releases/download/v0.1.7/mirror-clone.tar.gz".to_string(), - ttl: 3, + retry_limit: 3, } } @@ -434,7 +432,7 @@ mod tests { storage: "sjtug-internal", origin: "https://github.com/sjtug".to_string(), path: "mirror-clone/releases/download/v0.1.7/mirror-clone-2333.tar.gz".to_string(), - ttl: 3, + retry_limit: 3, } } @@ -443,7 +441,7 @@ mod tests { storage: "sjtug-internal", origin: "https://github.com/sjtug".to_string(), path: "mirror-clone/releases/download/v0.1.7/forbidden/mirror-clone.tar.gz".to_string(), - ttl: 3, + retry_limit: 3, } } @@ -456,7 +454,7 @@ mod tests { assert_eq!(response.status(), Status::MovedPermanently); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.cached(&config)] + vec![object.cached_url(&config).as_str()] ); } @@ -468,7 +466,7 @@ mod tests { assert_eq!(response.status(), Status::MovedPermanently); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.cached(&config)] + vec![object.cached_url(&config).as_str()] ); } @@ -481,7 +479,7 @@ mod tests { assert_eq!(response.status(), Status::Found); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.upstream()] + vec![object.upstream_url().as_str()] ); } @@ -494,7 +492,7 @@ mod tests { assert_eq!(response.status(), Status::Found); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.upstream()] + vec![object.upstream_url().as_str()] ); } @@ -507,7 +505,7 @@ mod tests { assert_eq!(response.status(), Status::MovedPermanently); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.upstream()] + vec![object.upstream_url().as_str()] ); } @@ -520,7 +518,7 @@ mod tests { assert_eq!(response.status(), Status::MovedPermanently); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.upstream()] + vec![object.upstream_url().as_str()] ); } @@ -532,13 +530,13 @@ mod tests { storage: "sjtug-internal", origin: "https://github.com/sjtug".to_string(), path: "mirror-clone/releases/download/v0.1.7/mirror%2B%2B%2B-clone.tar.gz".to_string(), - ttl: 3, + retry_limit: 3, }; let response = client.head(object.root_path()).dispatch().await; assert_eq!(response.status(), Status::Found); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.upstream()] + vec![object.upstream_url().as_str()] ); } @@ -550,7 +548,7 @@ mod tests { storage: "sjtug-internal", origin: "https://github.com/sjtug".to_string(), path: "mirror-clone/releases/download/v0.1.7/.mirror%2B%2B%2B-clone.tar.gz".to_string(), - ttl: 3, + retry_limit: 3, }; let response = client.head(object.root_path()).dispatch().await; assert_eq!(response.status(), Status::NotFound); @@ -566,13 +564,13 @@ mod tests { path: "mirror-clone/releases/download/v0.1.7/mirror-clone.tar.gz?ci=233333&ci2=23333333" .to_string(), - ttl: 3, + retry_limit: 3, }; let response = client.get(object.root_path()).dispatch().await; assert_eq!(response.status(), Status::Found); assert_eq!( response.headers().get("Location").collect::>(), - vec![&object.upstream()] + vec![object.upstream_url().as_str()] ); } @@ -583,15 +581,15 @@ mod tests { assert!(!flutter_allow(&config, "releases/releases_linux.json")); assert!(flutter_allow( &config, - "releases/stable/linux/flutter_linux_1.17.0-stable.tar.xz" + "releases/stable/linux/flutter_linux_1.17.0-stable.tar.xz", )); assert!(flutter_allow( &config, - "flutter/069b3cf8f093d44ec4bae1319cbfdc4f8b4753b6/android-arm/artifacts.zip" + "flutter/069b3cf8f093d44ec4bae1319cbfdc4f8b4753b6/android-arm/artifacts.zip", )); assert!(flutter_allow( &config, - "flutter/fonts/03bdd42a57aff5c496859f38d29825843d7fe68e/fonts.zip" + "flutter/fonts/03bdd42a57aff5c496859f38d29825843d7fe68e/fonts.zip", )); assert!(!flutter_allow(&config, "flutter/coverage/lcov.info")); } @@ -600,11 +598,11 @@ mod tests { fn test_task_override() { let mut task = Task { storage: "flutter_infra", - ttl: 233, + retry_limit: 233, origin: "https://storage.flutter-io.cn/".to_string(), path: "test".to_string(), }; - task.to_download_task(&[EndpointOverride { + task.apply_override(&[EndpointOverride { name: "flutter".to_string(), pattern: "https://storage.flutter-io.cn/".to_string(), replace: "https://storage.googleapis.com/".to_string(), @@ -620,7 +618,7 @@ mod tests { storage: "pytorch-wheels", origin: "https://download.pytorch.org/whl".to_string(), path: "torch_stable.html".to_string(), - ttl: 3, + retry_limit: 3, }; let response = client.head(object.root_path()).dispatch().await; assert_eq!(response.status(), Status::Ok); @@ -632,15 +630,15 @@ mod tests { config.github_release.allow.push("sjtug/lug/".to_string()); assert!(github_release_allow( &config, - "sjtug/lug/releases/download/v0.0.0/test.txt" + "sjtug/lug/releases/download/v0.0.0/test.txt", )); assert!(!github_release_allow( &config, - "sjtug/lug/2333/releases/download/v0.0.0/test.txt" + "sjtug/lug/2333/releases/download/v0.0.0/test.txt", )); assert!(!github_release_allow( &config, - "sjtug/lug2/releases/download/v0.0.0/test.txt" + "sjtug/lug2/releases/download/v0.0.0/test.txt", )); } } diff --git a/src/storage.rs b/src/storage.rs index c48597a..837c376 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,12 +1,13 @@ //! S3 storage backend. -use crate::error::{Error, Result}; +use std::time::Duration; use rusoto_core::credential::{AwsCredentials, StaticProvider}; use rusoto_core::Region; use rusoto_s3::{S3Client, S3}; -use std::time::Duration; use tokio::time::timeout; +use crate::error::{Error, Result}; + /// SJTU JCloud S3 region. fn jcloud_region() -> Region { Region::Custom { @@ -44,23 +45,27 @@ pub async fn stream_to_s3( ) -> Result { let s3_client = get_s3_client(); - let mut req = rusoto_s3::PutObjectRequest::default(); - req.body = Some(stream); - req.bucket = s3_bucket.to_string(); - req.key = path.to_string(); - req.content_length = Some(content_length as i64); + let req = rusoto_s3::PutObjectRequest { + body: Some(stream), + bucket: s3_bucket.to_string(), + key: path.to_string(), + content_length: Some(content_length as i64), + ..Default::default() + }; Ok(s3_client.put_object(req).await?) } /// Check whether authenticated S3 storage is available. pub async fn check_s3(bucket: &str) -> Result<()> { - Ok(timeout(Duration::from_secs(1), async move { + timeout(Duration::from_secs(1), async move { let s3_client = get_s3_client(); - let mut req = rusoto_s3::ListObjectsRequest::default(); - req.bucket = bucket.to_string(); + let req = rusoto_s3::ListObjectsRequest { + bucket: bucket.to_string(), + ..Default::default() + }; s3_client.list_objects(req).await?; Ok::<(), Error>(()) }) .await - .map_err(|err| Error::CustomError(format!("failed to check s3 storage {:?}", err)))??) + .map_err(|err| Error::CustomError(format!("failed to check s3 storage {:?}", err)))? } diff --git a/src/utils.rs b/src/utils.rs index 3d4d2b1..af04141 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,6 +1,3 @@ -use crate::common::{Config, IntelMission, IntelObject, IntelResponse, Task}; -use crate::error::{Error, Result}; - use std::io::Cursor; use futures::stream::TryStreamExt; @@ -11,6 +8,10 @@ use rocket::{ Response, }; use tokio_util::compat::FuturesAsyncReadCompatExt; +use url::Url; + +use crate::common::{Config, IntelMission, IntelObject, IntelResponse, Task}; +use crate::error::{Error, Result}; impl Task { /// Resolve a task. @@ -24,26 +25,24 @@ impl Task { ) -> Result { mission.metrics.resolve_counter.inc(); let req = if head { - mission.client.head(&self.cached(config)) + mission.client.head(self.cached_url(config)) } else { - mission.client.get(&self.cached(config)) + mission.client.get(self.cached_url(config)) }; - if let Ok(resp) = req.send().await { - if resp.status().is_success() { - return Ok(IntelObject::Cached { task: self, resp }); - } else { + match req.send().await { + Ok(resp) if resp.status().is_success() => Ok(IntelObject::Cached { task: self, resp }), + _ => { if let Some(tx) = &mission.tx { mission.metrics.task_in_queue.inc(); - // TODO this may block if the queue is full, which is not good. - tx - .clone() + // TODO this may block if the queue is full, which is not good + tx.clone() .send(self.clone()) .await .map_err(|_| Error::SendError(()))?; } + Ok(IntelObject::Origin { task: self }) } } - Ok(IntelObject::Origin { task: self }) } /// Resolve a task. @@ -75,33 +74,32 @@ impl Task { } /// Always resolves to upstream. Neither returns cache nor schedules a download. - pub fn resolve_upstream(self) -> IntelObject { + pub const fn resolve_upstream(self) -> IntelObject { IntelObject::Origin { task: self } } } impl IntelObject { /// Extract original task from the object. - pub fn task(&self) -> &Task { + pub const fn task(&self) -> &Task { match self { - IntelObject::Cached { task, .. } => task, - IntelObject::Origin { task, .. } => task, + Self::Cached { task, .. } | Self::Origin { task, .. } => task, } } /// Get URL target. - pub fn target(&self, config: &crate::common::Config) -> String { + pub fn target_url(&self, config: &Config) -> Url { match self { - IntelObject::Cached { task, .. } => task.cached(config), - IntelObject::Origin { task } => task.upstream(), + Self::Cached { task, .. } => task.cached_url(config), + Self::Origin { task } => task.upstream_url(), } } /// Respond with redirection. - pub fn redirect(self, config: &crate::common::Config) -> Redirect { + pub fn redirect(self, config: &Config) -> Redirect { match &self { - IntelObject::Cached { .. } => Redirect::moved(self.target(config)), - IntelObject::Origin { .. } => Redirect::found(self.target(config)), + Self::Cached { .. } => Redirect::moved(self.target_url(config).to_string()), + Self::Origin { .. } => Redirect::found(self.target_url(config).to_string()), } } @@ -114,8 +112,8 @@ impl IntelObject { config: &Config, ) -> Result> { let task = self.task(); - let upstream = task.upstream(); - let resp = intel_mission.client.get(&upstream).send().await?; + let upstream = task.upstream_url(); + let resp = intel_mission.client.get(upstream).send().await?; if resp.status().is_success() { if let Some(content_length) = resp.content_length() { @@ -134,7 +132,6 @@ impl IntelObject { } /// Pass status code from upstream to new response. - /// TODO extract to adhoc trait or method? fn set_status(intel_response: &mut ResponseBuilder, response: &reqwest::Response) { let status = response.status(); // special case for NGINX 499 @@ -153,7 +150,7 @@ impl IntelObject { intel_mission: &IntelMission, ) -> Result> { match self { - IntelObject::Cached { resp, .. } => { + Self::Cached { resp, .. } => { let mut intel_response = Response::build(); if let Some(content_length) = resp.content_length() { intel_response.raw_header("content-length", content_length.to_string()); @@ -172,8 +169,8 @@ impl IntelObject { ); Ok(intel_response.finalize().into()) } - IntelObject::Origin { ref task } => { - let resp = intel_mission.client.get(&task.upstream()).send().await?; + Self::Origin { ref task } => { + let resp = intel_mission.client.get(task.upstream_url()).send().await?; let mut intel_response = Response::build(); if let Some(content_length) = resp.content_length() { intel_response.raw_header("content-length", content_length.to_string()); @@ -202,18 +199,17 @@ impl IntelObject { intel_mission: &IntelMission, config: &Config, ) -> Result> { - // TODO if let match &self { - IntelObject::Cached { resp, .. } => { + Self::Cached { resp, .. } => { if let Some(content_length) = resp.content_length() { if content_length <= size_kb * 1024 { debug!("{} <= {}, direct stream", content_length, size_kb * 1024); - return Ok(self.reverse_proxy(intel_mission).await?.into()); + return self.reverse_proxy(intel_mission).await; } } Ok(self.redirect(config).into()) } - IntelObject::Origin { .. } => Ok(self.redirect(config).into()), + Self::Origin { .. } => Ok(self.redirect(config).into()), } } } @@ -229,10 +225,10 @@ pub fn not_found(req: &rocket::Request) -> Response<'static> { /// Hint user to redirect to the S3 index page. pub fn no_route_for(mut route: &str) -> Response<'static> { let mut resp = Response::build(); - if route.ends_with("/") { + if route.ends_with('/') { route = &route[..route.len() - 1]; } - if route.starts_with("/") { + if route.starts_with('/') { route = &route[1..]; } let body = format!( From 077fea108de8f981b039fdfb46f255d18cbf8e5d Mon Sep 17 00:00:00 2001 From: LightQuantum Date: Tue, 30 Aug 2022 10:08:14 +0800 Subject: [PATCH 5/6] refactor: migrate from slog to tracing --- Cargo.lock | 539 +++++++++++++++++++++++++++++++++++++---------- Cargo.toml | 10 +- src/artifacts.rs | 107 ++++------ src/main.rs | 76 +++++-- src/utils.rs | 1 + 5 files changed, 532 insertions(+), 201 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aeb19aa..3c37d18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,168 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "actix-codec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "log", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-http" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f9ffb6db08c1c3a1f4aef540f1a63193adc73c4fbd40b75a95fc8c5258f6e51" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "ahash", + "base64", + "bitflags", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand 0.8.5", + "sha1 0.10.2", + "smallvec", + "tracing", +] + +[[package]] +name = "actix-router" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb60846b52c118f2f04a56cc90880a274271c489b2498623d58176f8ca21fa80" +dependencies = [ + "bytestring", + "firestorm", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "num_cpus", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a27e8fe9ba4ae613c21f677c2cfaf0696c3744030c6f485b34634e502d6bb379" +dependencies = [ + "actix-codec", + "actix-http", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "ahash", + "bytes", + "bytestring", + "cfg-if", + "derive_more", + "encoding_rs", + "futures-core", + "futures-util", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.3.14", + "url", +] + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -21,10 +183,13 @@ dependencies = [ ] [[package]] -name = "arc-swap" -version = "1.5.1" +name = "ansi_term" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] [[package]] name = "async-trait" @@ -96,6 +261,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.11.0" @@ -108,6 +282,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +[[package]] +name = "bytestring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b6a75fd3048808ef06af5cd79712be8111960adaf89d90250974b38fc3928a" +dependencies = [ + "bytes", +] + [[package]] name = "cc" version = "1.0.73" @@ -139,6 +322,12 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "cookie" version = "0.15.0-dev" @@ -203,6 +392,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-mac" version = "0.11.1" @@ -213,6 +412,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn", +] + [[package]] name = "devise" version = "0.3.0" @@ -252,6 +464,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -317,6 +539,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "firestorm" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" + [[package]] name = "fnv" version = "1.0.7" @@ -453,6 +681,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "getrandom" version = "0.2.7" @@ -517,7 +755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ "crypto-mac", - "digest", + "digest 0.9.0", ] [[package]] @@ -662,6 +900,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.4.0" @@ -674,6 +918,24 @@ version = "0.2.132" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" +[[package]] +name = "local-channel" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" + [[package]] name = "lock_api" version = "0.4.8" @@ -693,6 +955,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.9" @@ -705,8 +976,8 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" dependencies = [ - "block-buffer", - "digest", + "block-buffer 0.9.0", + "digest 0.9.0", "opaque-debug", ] @@ -752,15 +1023,17 @@ dependencies = [ "rusoto_core", "rusoto_s3", "serde", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", "tap", "tempdir", "thiserror", "tokio", "tokio-util", + "tracing", + "tracing-actix-web", + "tracing-appender", + "tracing-bunyan-formatter", + "tracing-log", + "tracing-subscriber", "url", ] @@ -1180,6 +1453,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.27" @@ -1365,7 +1647,7 @@ dependencies = [ "base64", "bytes", "chrono", - "digest", + "digest 0.9.0", "futures", "hex", "hmac", @@ -1400,12 +1682,6 @@ dependencies = [ "semver 1.0.13", ] -[[package]] -name = "rustversion" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" - [[package]] name = "ryu" version = "1.0.11" @@ -1524,6 +1800,17 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf2781a4ca844dd4f9b608a1791eea19830df0ad3cdd9988cd05f1c66ccb63a" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sha1_smol" version = "1.0.0" @@ -1536,13 +1823,22 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.1.0" @@ -1567,74 +1863,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-async" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] - -[[package]] -name = "slog-envlogger" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" -dependencies = [ - "log", - "regex", - "slog", - "slog-async", - "slog-scope", - "slog-stdlog", - "slog-term", -] - -[[package]] -name = "slog-scope" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" -dependencies = [ - "arc-swap", - "lazy_static", - "slog", -] - -[[package]] -name = "slog-stdlog" -version = "4.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" -dependencies = [ - "log", - "slog", - "slog-scope", -] - -[[package]] -name = "slog-term" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" -dependencies = [ - "atty", - "slog", - "term", - "thread_local", - "time 0.3.14", -] - [[package]] name = "smallvec" version = "1.9.0" @@ -1705,7 +1933,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1", + "sha1 0.6.1", "syn", ] @@ -1732,12 +1960,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - [[package]] name = "tap" version = "1.0.1" @@ -1768,17 +1990,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "thiserror" version = "1.0.33" @@ -1818,7 +2029,7 @@ dependencies = [ "libc", "standback", "stdweb", - "time-macros 0.1.1", + "time-macros", "version_check", "winapi", ] @@ -1832,7 +2043,6 @@ dependencies = [ "itoa", "libc", "num_threads", - "time-macros 0.2.4", ] [[package]] @@ -1845,12 +2055,6 @@ dependencies = [ "time-macros-impl", ] -[[package]] -name = "time-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" - [[package]] name = "time-macros-impl" version = "0.1.2" @@ -1958,8 +2162,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ "cfg-if", + "log", "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-actix-web" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0b2cd6e5074ff67679a0ff68bc6333be78d29910ba7cd295d948aa5ff152032" +dependencies = [ + "actix-web", + "pin-project", + "tracing", + "tracing-futures", + "uuid", +] + +[[package]] +name = "tracing-appender" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" +dependencies = [ + "crossbeam-channel", + "time 0.3.14", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-bunyan-formatter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a788f2119fde477cd33823330c14004fa8cdac6892fd6f12181bbda9dbf14fc9" +dependencies = [ + "gethostname", + "log", + "serde", + "serde_json", + "time 0.3.14", + "tracing", "tracing-core", + "tracing-log", + "tracing-subscriber", ] [[package]] @@ -1969,6 +2227,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +dependencies = [ + "ansi_term", + "matchers", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -2040,6 +2338,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" +dependencies = [ + "getrandom", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 3e26faf..9708add 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,14 +21,16 @@ rocket_codegen = { git = "https://github.com/SergioBenitez/Rocket", rev = "93e62 rusoto_core = "0.48.0" rusoto_s3 = "0.48.0" serde = "1.0" -slog = "2.5" -slog-async = "2.5" -slog-envlogger = "2.2" -slog-term = "2.6" tap = "1.0" thiserror = "1.0" tokio = { version = "1.0", features = ["full"] } tokio-util = { version = "0.7", features = ["compat"] } +tracing = "0.1" +tracing-actix-web = "0.6" +tracing-appender = "0.2" +tracing-bunyan-formatter = "0.3" +tracing-log = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = "2.2" [dev-dependencies] diff --git a/src/artifacts.rs b/src/artifacts.rs index 937518b..33cab3a 100644 --- a/src/artifacts.rs +++ b/src/artifacts.rs @@ -7,13 +7,11 @@ use std::sync::atomic::AtomicUsize; use std::sync::Arc; use std::task::{Context, Poll}; -use bytes::BytesMut; +use bytes::{Bytes, BytesMut}; use futures::{Stream, TryStreamExt}; use futures_util::StreamExt; use pin_project::pin_project; use reqwest::{Client, Response}; -use rocket::http::hyper::Bytes; -use slog::{debug, info, Logger, o, warn}; use tap::Pipe; use tokio::fs::{self, File, OpenOptions}; use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter}; @@ -21,6 +19,7 @@ use tokio::sync::mpsc::{unbounded_channel, Receiver}; use tokio::sync::Mutex; use tokio::sync::Semaphore; use tokio_util::codec; +use tracing::{debug, info, warn}; use url::Url; use crate::common::{Config, Metrics, Task}; @@ -31,12 +30,11 @@ type IOResult = std::result::Result; /// Convert reqwest resp stream to io result stream. fn into_io_stream( - stream: impl Stream>, - logger: Logger, -) -> impl Stream { + stream: impl Stream>, +) -> impl Stream { stream.map(move |x| { x.map_err(|err| { - warn!(logger, "failed to receive data: {:?}", err); + warn!("failed to receive data: {:?}", err); std::io::Error::new(std::io::ErrorKind::Other, err) }) }) @@ -83,16 +81,13 @@ impl FileWrapper { } /// Convert this file into a stream of bytes. - async fn into_bytes_stream(mut self, logger: Logger) -> Result> { + async fn into_bytes_stream(mut self) -> Result> { // remove file on disk, but we could still read it let mut f = self.f.take().unwrap(); f.flush().await?; let mut f = f.into_inner(); if let Err(err) = fs::remove_file(&self.path).await { - warn!( - logger, - "failed to remove cache file: {:?} {:?}", err, self.path - ); + warn!("failed to remove cache file: {:?} {:?}", err, self.path); } f.seek(std::io::SeekFrom::Start(0)).await?; Ok( @@ -116,9 +111,8 @@ impl Drop for FileWrapper { /// The old stream is consumed and a new stream with the same contents is returned. /// It can be used to download large files from the network. async fn into_file_stream( - mut stream: impl Stream + Unpin, - logger: Logger, -) -> Result> { + mut stream: impl Stream + Unpin, +) -> Result> { let path = format!( "/mnt/cache/{}", FILE_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) @@ -128,7 +122,7 @@ async fn into_file_stream( let v = v?; file.as_mut().write_all(&v).await?; } - Ok(file.into_bytes_stream(logger).await?) + file.into_bytes_stream().await } /// Convert a stream of bytes to a memory-backed one. @@ -137,8 +131,8 @@ async fn into_file_stream( /// It can be used to download small files from the network. async fn into_memory_stream( content_length: usize, - mut stream: impl Stream + Unpin, -) -> Result> { + mut stream: impl Stream + Unpin, +) -> Result> { let mut result = Vec::with_capacity(content_length); while let Some(v) = stream.next().await { let v = v?; @@ -151,12 +145,7 @@ async fn into_memory_stream( /// /// This function does the actual caching part. /// It's called in `download_artifact`, which does something like concurrency control and retries. -async fn process_task( - task: Task, - client: Client, - config: &Config, - logger: slog::Logger, -) -> Result<()> { +async fn process_task(task: Task, client: Client, config: &Config) -> Result<()> { if client .head(task.cached_url(config)) .send() @@ -164,12 +153,11 @@ async fn process_task( .status() .is_success() { - info!(logger, "already exists"); + info!("already exists"); return Ok(()); } - let (content_length, stream) = - stream_from_url(client, task.upstream_url(), config, logger.clone()).await?; - info!(logger, "get length={}", content_length); + let (content_length, stream) = stream_from_url(client, task.upstream_url(), config).await?; + info!("get length={}", content_length); let key = task.s3_key()?; let result = stream_to_s3( &key, @@ -177,9 +165,9 @@ async fn process_task( rusoto_s3::StreamingBody::new(stream), &config.s3.bucket, ) - .await?; - info!(logger, "upload to bucket"); - debug!(logger, "{:?}", result); + .await?; + info!("upload to bucket"); + debug!("{:?}", result); Ok(()) } @@ -190,9 +178,9 @@ enum Either { } impl Stream for Either - where - T: Stream, - U: Stream, +where + T: Stream, + U: Stream, { type Item = O; @@ -210,30 +198,29 @@ impl Stream for Either async fn into_stream( resp: Response, config: &Config, - logger: Logger, -) -> Result<(u64, impl Stream + Send + Sync)> { +) -> Result<(u64, impl Stream + Send + Sync)> { if let Some(content_length) = resp.content_length() { if content_length > config.ignore_threshold_mb * 1024 * 1024 { Err(Error::TooLarge(())) } else { - let io_stream = resp.bytes_stream().pipe(|s| into_io_stream(s, logger.clone())); + let io_stream = resp.bytes_stream().pipe(into_io_stream); if content_length > config.file_threshold_mb * 1024 * 1024 { - info!(logger, "stream mode: file backend"); - let stream = io_stream.pipe(|s| into_file_stream(s, logger)).await?; + info!("stream mode: file backend"); + let stream = io_stream.pipe(into_file_stream).await?; Ok((content_length, Either::Left(Either::Left(stream)))) } else if content_length > 1024 * 1024 { - info!(logger, "stream mode: memory cache"); - let stream = io_stream.pipe(|s| - into_memory_stream(content_length as usize, s)) + info!("stream mode: memory cache"); + let stream = io_stream + .pipe(|s| into_memory_stream(content_length as usize, s)) .await?; Ok((content_length, Either::Left(Either::Right(stream)))) } else { - info!(logger, "stream mode: direct copy"); + info!("stream mode: direct copy"); Ok((content_length, Either::Right(Either::Left(io_stream)))) } } } else { - info!(logger, "stream mode: direct copy"); + info!("stream mode: direct copy"); let resp = resp.bytes().await?; Ok(( resp.len() as u64, @@ -247,14 +234,13 @@ async fn stream_from_url( client: Client, url: Url, config: &Config, - logger: Logger, -) -> Result<(u64, impl Stream + Send + Sync)> { +) -> Result<(u64, impl Stream + Send + Sync)> { let response = client.get(url).send().await?; let status = response.status(); if !status.is_success() { return Err(Error::HTTPError(status)); } - into_stream(response, config, logger).await + into_stream(response, config).await } /// Main artifact download task. @@ -263,7 +249,6 @@ async fn stream_from_url( pub async fn download_artifacts( mut rx: Receiver, client: Client, - logger: Logger, config: Arc, metrics: Arc, ) { @@ -294,9 +279,6 @@ pub async fn download_artifacts( continue; } - // TODO Oh I see why making logger global is blocked. What about replace slog with tracing? - let logger = logger.new(o!("storage" => task.storage, "origin" => task.origin.clone(), "path" => task.path.clone())); - if task.retry_limit == 0 { // The task has been retried too many times. Skip it. continue; @@ -308,13 +290,13 @@ pub async fn download_artifacts( // Deduplicate tasks. let mut processing_task = processing_task.lock().await; if processing_task.contains(&task_hash) { - info!(logger, "already processing, continue to next task"); + info!("already processing, continue to next task"); continue; } processing_task.insert(task_hash.clone()); } - info!(logger, "start download"); + info!("start download"); metrics.download_counter.inc(); // Wait for concurrency permit. @@ -332,14 +314,14 @@ pub async fn download_artifacts( let _permit = permit; let mut task_new = task.clone(); - info!(logger, "begin stream"); - let task_fut = process_task(task, client, &config, logger.clone()); + info!("begin stream"); + let task_fut = process_task(task, client, &config); let task_fut = tokio::time::timeout( std::time::Duration::from_secs(config.download_timeout), task_fut, ); if let Err(err) = task_fut.await.unwrap_or(Err(Error::Timeout(()))) { - warn!(logger, "{:?}, ttl={}", err, task_new.retry_limit); + warn!("{:?}, ttl={}", err, task_new.retry_limit); task_new.retry_limit -= 1; metrics.failed_download_counter.inc(); @@ -361,31 +343,22 @@ pub async fn download_artifacts( }); } - info!(logger, "artifact download stop"); + info!("artifact download stop"); } #[cfg(test)] mod tests { - use slog::Drain; use tempdir::TempDir; use super::*; - fn create_test_logger() -> Logger { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::FullFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - Logger::root(drain, o!()) - } - #[tokio::test] async fn test_overlay_file_create() { - let logger = create_test_logger(); let tmp_dir = TempDir::new("intel").unwrap(); let path = tmp_dir.path().join("test.bin"); let mut wrapper = FileWrapper::open(&path).await.unwrap(); wrapper.as_mut().write_all(b"233333333").await.unwrap(); - let mut stream = wrapper.into_bytes_stream(logger).await.unwrap(); + let mut stream = wrapper.into_bytes_stream().await.unwrap(); assert_eq!(&stream.next().await.unwrap().unwrap(), "233333333"); } } diff --git a/src/main.rs b/src/main.rs index 9d81232..378452a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,9 +6,12 @@ use std::sync::Arc; use prometheus::{Encoder, TextEncoder}; use reqwest::{Client, ClientBuilder}; use rocket::State; -use slog::{info, warn}; -use slog::{o, Drain}; use tokio::sync::mpsc::channel; +use tracing::{info, warn}; +use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; +use tracing_log::LogTracer; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::{fmt, EnvFilter, Registry}; use artifacts::download_artifacts; use browse::list; @@ -31,13 +34,45 @@ mod storage; mod utils; /// Create a logger with styled output, env-filter, and async logging. -/// TODO what about a global logger? There's no reason to pass it around. -fn create_logger() -> slog::Logger { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::FullFormat::new(decorator).build().fuse(); - let drain = slog_envlogger::new(drain); - let drain = slog_async::Async::new(drain).chan_size(1024).build().fuse(); - slog::Logger::root(drain, o!()) +fn setup_log() -> impl Drop { + let registry = Registry::default().with(EnvFilter::from_default_env()); + let (writer, guard) = tracing_appender::non_blocking(std::io::stdout()); + + let rust_log_format = std::env::var("RUST_LOG_FORMAT") + .unwrap_or_default() + .to_lowercase(); + + let (json, after) = + match (rust_log_format.as_str(), cfg!(debug_assertions)) { + ("plain", _) => (false, None), + ("json", _) => (true, None), + ("", dev) => (!dev, None), // release defaults to json and debug to plain + (format, dev) => ( + !dev, + Some(move || { + warn!( + "RUST_LOG_FORMAT is set to '{}', but mirror-intel is in {} mode. Using '{}'", + format, if dev {"debug"} else {"release"}, if dev { "plain" } else { "json" } + ); + }), + ), + }; + + if json { + tracing::subscriber::set_global_default(registry.with(JsonStorageLayer).with( + BunyanFormattingLayer::new("mirror-intel".to_string(), writer), + )) + .expect("Unable to set logger"); + } else { + tracing::subscriber::set_global_default( + registry.with(fmt::Layer::default().pretty().with_writer(writer)), + ) + .expect("Unable to set logger"); + }; + if let Some(after) = after { + after(); + } + guard } /// Metrics endpoint. @@ -54,21 +89,28 @@ pub async fn metrics(intel_mission: State<'_, IntelMission>) -> Result> #[launch] async fn rocket() -> rocket::Rocket { - let logger = create_logger(); + LogTracer::init().unwrap(); + let _guard = setup_log(); + let rocket = rocket::ignite(); let figment = rocket.figment(); let config: Config = figment.extract().expect("config"); - info!(logger, "checking if bucket is available..."); + info!("checking if bucket is available..."); // check if credentials are set and we have permissions - if let Err(error) = check_s3(&config.s3.bucket).await { - warn!(logger, "s3 storage backend not available, but not running in read-only mode"; "error" => format!("{:?}", error)); - // config.read_only = true; + if !config.read_only { + if let Err(error) = check_s3(&config.s3.bucket).await { + warn!( + ?error, + "s3 storage backend not available, but not running in read-only mode" + ); + // config.read_only = true; + } } - info!(logger, "{:?}", config); + info!(?config, "config loaded"); - info!(logger, "starting server..."); + info!("starting server..."); let metrics = Arc::new(Metrics::default()); let metrics_download = metrics.clone(); @@ -81,7 +123,7 @@ async fn rocket() -> rocket::Rocket { // Spawn caching future. tokio::spawn(async move { - download_artifacts(rx, Client::new(), logger, config_download, metrics_download).await; + download_artifacts(rx, Client::new(), config_download, metrics_download).await; }); tx diff --git a/src/utils.rs b/src/utils.rs index af04141..7d5cda2 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,6 +8,7 @@ use rocket::{ Response, }; use tokio_util::compat::FuturesAsyncReadCompatExt; +use tracing::debug; use url::Url; use crate::common::{Config, IntelMission, IntelObject, IntelResponse, Task}; From f68ca2d54a3d77e8dab53fd677950936bda690a4 Mon Sep 17 00:00:00 2001 From: LightQuantum Date: Wed, 31 Aug 2022 13:26:47 +0800 Subject: [PATCH 6/6] refactor: artifact download --- src/artifacts.rs | 238 +++++++++++++++++++++++++++++++---------------- 1 file changed, 158 insertions(+), 80 deletions(-) diff --git a/src/artifacts.rs b/src/artifacts.rs index 33cab3a..5e102d0 100644 --- a/src/artifacts.rs +++ b/src/artifacts.rs @@ -1,5 +1,6 @@ //! Artifact download implementation. +use std::borrow::Cow; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -15,11 +16,10 @@ use reqwest::{Client, Response}; use tap::Pipe; use tokio::fs::{self, File, OpenOptions}; use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter}; -use tokio::sync::mpsc::{unbounded_channel, Receiver}; -use tokio::sync::Mutex; -use tokio::sync::Semaphore; +use tokio::sync::mpsc::{unbounded_channel, Receiver, UnboundedSender}; +use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore}; use tokio_util::codec; -use tracing::{debug, info, warn}; +use tracing::{debug, info, instrument, warn}; use url::Url; use crate::common::{Config, Metrics, Task}; @@ -141,11 +141,152 @@ async fn into_memory_stream( Ok(futures::stream::iter(vec![Ok(Bytes::from(result))])) } +/// Download context for a single artifact. +struct DownloadCtx<'a> { + task: Task, + config: Cow<'a, Arc>, + metrics: Cow<'a, Arc>, + client: Cow<'a, Client>, + processing_task: Cow<'a, Arc>>>, + fail_tx: Cow<'a, UnboundedSender>, + extra: DownloadStageExtra<'a>, +} + +/// Extra context for different stages of the download process. +enum DownloadStageExtra<'a> { + Pre { + sem: &'a Arc, + }, + On { + _permit: OwnedSemaphorePermit, + task_hash: Url, + }, +} + +impl<'a> DownloadCtx<'a> { + /// Do prepare work before actual download. + /// + /// This method checks whether there's an ongoing download for the same artifact, + /// limit the number of concurrent downloads, and skip the download if retry_limit is reached. + /// + /// After the prepare work has been done, you must call `spawn()` immediately to start the + /// download, or further download tasks will be blocked. + /// + /// Returns `None` if the download should be skipped. + #[instrument(skip(self), fields(storage = self.task.storage, origin = self.task.origin, path = self.task.path))] + pub async fn prepare(self) -> Option> { + self.metrics.task_in_queue.dec(); + + // We need to ensure that the total count of pending tasks doesn't exceed the set limit. + // The income `rx` is already bounded by `max_pending_task`, so it's the retried tasks that + // are the problem. + // If a task is retried and current pending queue is full, this will randomly ignore a + // retried task or an incoming task. + + // TODO I don't think the current double queue design is good. We need to prio income over + // retries, i.e. income overtakes retries. + if self.metrics.task_in_queue.get() > self.config.max_pending_task as i64 { + return None; + } + + if self.task.retry_limit == 0 { + // The task has been retried too many times. Skip it. + return None; + } + + let task_hash = self.task.upstream_url(); + + { + // Deduplicate tasks. + let mut processing_task = self.processing_task.lock().await; + if processing_task.contains(&task_hash) { + info!("already processing, continue to next task"); + return None; + } + processing_task.insert(task_hash.clone()); + } + + match self.extra { + DownloadStageExtra::Pre { sem } => { + // Wait for concurrency permit. + let permit = Arc::clone(sem).acquire_owned().await.unwrap(); + + let client = Cow::Owned(self.client.into_owned()); + let processing_task = Cow::Owned(self.processing_task.into_owned()); + let metrics = Cow::Owned(self.metrics.into_owned()); + let fail_tx = Cow::Owned(self.fail_tx.into_owned()); + let config = Cow::Owned(self.config.into_owned()); + + Some(DownloadCtx { + task: self.task, + config, + metrics, + client, + processing_task, + fail_tx, + extra: DownloadStageExtra::On { + _permit: permit, + task_hash, + }, + }) + } + DownloadStageExtra::On { .. } => unreachable!(), + } + } +} + +impl DownloadCtx<'static> { + /// Spawn the download task. + pub fn spawn(self) { + tokio::spawn(self.download()); + } + /// Actual download future. + #[instrument(skip(self), fields(storage = self.task.storage, origin = self.task.origin, path = self.task.path))] + async fn download(self) { + info!("start download"); + self.metrics.download_counter.inc(); + + self.metrics.task_download.inc(); + + let mut task_new = self.task.clone(); + + info!("begin stream"); + let config = self.config.into_owned(); + let task_fut = cache_task(self.task, self.client.into_owned(), &config); + let task_fut = tokio::time::timeout( + std::time::Duration::from_secs(config.download_timeout), + task_fut, + ); + if let Err(err) = task_fut.await.unwrap_or(Err(Error::Timeout(()))) { + warn!("{:?}, ttl={}", err, task_new.retry_limit); + task_new.retry_limit -= 1; + self.metrics.failed_download_counter.inc(); + + if !matches!(err, Error::HTTPError(_)) && !matches!(err, Error::TooLarge(_)) { + self.fail_tx.send(task_new).unwrap(); + self.metrics.task_in_queue.inc(); + } + }; + + { + let mut processing_task = self.processing_task.lock().await; + match self.extra { + DownloadStageExtra::On { task_hash, .. } => { + processing_task.remove(&task_hash); + } + DownloadStageExtra::Pre { .. } => unreachable!(), + } + } + + self.metrics.task_download.dec(); + } +} + /// Cache a task. /// /// This function does the actual caching part. /// It's called in `download_artifact`, which does something like concurrency control and retries. -async fn process_task(task: Task, client: Client, config: &Config) -> Result<()> { +async fn cache_task(task: Task, client: Client, config: &Config) -> Result<()> { if client .head(task.cached_url(config)) .send() @@ -265,82 +406,19 @@ pub async fn download_artifacts( // Apply override rules on the task. task.apply_override(&config.endpoints.overrides); - metrics.task_in_queue.dec(); - - // We need to ensure that the total count of pending tasks doesn't exceed the set limit. - // The income `rx` is already bounded by `max_pending_task`, so it's the retried tasks that - // are the problem. - // If a task is retried and current pending queue is full, this will randomly ignore a - // retried task or an incoming task. - - // TODO I don't think the current double queue design is good. We need to prio income over - // retries, i.e. income overtakes retries. - if metrics.task_in_queue.get() > config.max_pending_task as i64 { - continue; + let ctx = DownloadCtx { + task, + config: Cow::Borrowed(&config), + metrics: Cow::Borrowed(&metrics), + client: Cow::Borrowed(&client), + processing_task: Cow::Borrowed(&processing_task), + fail_tx: Cow::Borrowed(&fail_tx), + extra: DownloadStageExtra::Pre { sem: &sem }, + }; + + if let Some(ctx) = ctx.prepare().await { + ctx.spawn(); } - - if task.retry_limit == 0 { - // The task has been retried too many times. Skip it. - continue; - } - - let task_hash = task.upstream_url(); - - { - // Deduplicate tasks. - let mut processing_task = processing_task.lock().await; - if processing_task.contains(&task_hash) { - info!("already processing, continue to next task"); - continue; - } - processing_task.insert(task_hash.clone()); - } - - info!("start download"); - metrics.download_counter.inc(); - - // Wait for concurrency permit. - let permit = Arc::clone(&sem).acquire_owned().await.unwrap(); - - let client = client.clone(); - let processing_task = processing_task.clone(); - let metrics = metrics.clone(); - let fail_tx = fail_tx.clone(); - let config = config.clone(); - - metrics.task_download.inc(); - // Spawn actual task download task. - tokio::spawn(async move { - let _permit = permit; - let mut task_new = task.clone(); - - info!("begin stream"); - let task_fut = process_task(task, client, &config); - let task_fut = tokio::time::timeout( - std::time::Duration::from_secs(config.download_timeout), - task_fut, - ); - if let Err(err) = task_fut.await.unwrap_or(Err(Error::Timeout(()))) { - warn!("{:?}, ttl={}", err, task_new.retry_limit); - task_new.retry_limit -= 1; - metrics.failed_download_counter.inc(); - - { - let mut processing_task = processing_task.lock().await; - processing_task.remove(&task_hash); - } - - if !matches!(err, Error::HTTPError(_)) && !matches!(err, Error::TooLarge(_)) { - fail_tx.send(task_new).unwrap(); - metrics.task_in_queue.inc(); - } - } else { - let mut processing_task = processing_task.lock().await; - processing_task.remove(&task_hash); - } - - metrics.task_download.dec(); - }); } info!("artifact download stop");