diff --git a/Cargo.lock b/Cargo.lock index 89185c9a7..43040f8d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.11" @@ -46,47 +57,48 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -94,9 +106,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" [[package]] name = "arc-cell" @@ -179,15 +191,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "1.2.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4707646259764ab59fd9a50e9de2e92c637b28b36285d6f6fa030e915fbd9" +checksum = "40ddbfb5db93d62521f47b3f223da0884a2f02741ff54cb9cda192a0e73ba08b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -228,9 +240,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.2.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4963ac9ff2d33a4231b3806c1c69f578f221a9cabb89ad2bde62ce2b442c8a7" +checksum = "75588e7ee5e8496eed939adac2035a6dbab9f7eb2acdd9ab2d31856dab6f3955" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -252,11 +264,11 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.24.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f522b68eb0294c59f7beb0defa30e84fed24ebc50ee219e111d6c33eaea96a8" +checksum = "91303c2fed0fc851059edd5a3c69cc139197bf41accbaab9cfd60b2410647a0e" dependencies = [ - "ahash", + "ahash 0.8.11", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -287,9 +299,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.21.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d70fb493f4183f5102d8a8d0cc9b57aec29a762f55c0e7bf527e0f7177bb408" +checksum = "5fd6a9b38fe2dcaa2422b42e2c667112872d03a83522533f8b4165fd2d9d4bd1" dependencies = [ "aws-credential-types", "aws-runtime", @@ -309,9 +321,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.21.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3f37549b3e38b7ea5efd419d4d7add6ea1e55223506eb0b4fef9d25e7cc90d" +checksum = "c99f342a364c8490b7715387244d2f7ebfc05f07bb6a0fc6e70b8e62b7078d06" dependencies = [ "aws-credential-types", "aws-runtime", @@ -331,9 +343,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.21.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2ff219a5d4b795cd33251c19dbe9c4b401f2b2cbe513e07c76ada644eaf34e" +checksum = "7207ca62206c93e470457babf0512d7b6b97170fdba929a2a2f5682f9f68407c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -482,9 +494,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44e7945379821074549168917e89e60630647e186a69243248f08c6d168b975a" +checksum = "c9ac79e9f3a4d576f3cd4a470a0275b138d9e7b11b1cd514a6858ae0a79dd5bb" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -503,7 +515,7 @@ dependencies = [ "once_cell", "pin-project-lite", "pin-utils", - "rustls 0.21.11", + "rustls 0.21.12", "serde", "serde_json", "tokio", @@ -513,9 +525,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc56a5c96ec741de6c5e6bf1ce6948be969d6506dfa9c39cffc284e31e4979b" +checksum = "04ec42c2f5c0e7796a2848dde4d9f3bf8ce12ccbb3d5aa40c52fa0cdd61a1c47" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -530,9 +542,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe14dceea1e70101d38fbf2a99e6a34159477c0fb95e68e05c66bd7ae4c3729" +checksum = "baf98d97bba6ddaba180f1b1147e202d8fe04940403a95a3f826c790f931bbd1" dependencies = [ "base64-simd", "bytes", @@ -565,9 +577,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a43b56df2c529fe44cb4d92bd64d0479883fb9608ff62daede4df5405381814" +checksum = "a807d90cd50a969b3d95e4e7ad1491fcae13c6e83948d8728363ecc09d66343a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -656,9 +668,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64-simd" @@ -697,6 +709,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.5.1" @@ -720,6 +744,63 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe5b10e214954177fb1dc9fbd20a1a2608fe99e6c832033bdc7cea287a20d77" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a8646f94ab393e43e8b35a2558b1624bed28b97ee09c5d15456e3c9463f46d" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.52", + "syn_derive", +] + +[[package]] +name = "byte-unit" +version = "5.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ac19bdf0b2665407c39d82dbc937e951e7e2001609f0fb32edd0af45a2d63e" +dependencies = [ + "rust_decimal", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -744,9 +825,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.95" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" [[package]] name = "cfg-if" @@ -754,6 +835,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "clap" version = "4.5.4" @@ -796,9 +883,9 @@ checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "combine" @@ -816,9 +903,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", ] @@ -1071,9 +1158,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1114,9 +1201,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "ff" @@ -1136,7 +1223,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -1148,9 +1235,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1177,6 +1264,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db0f0c49aba98a3b2578315766960bd242885ff672fd62610c5557cd6c6efe03" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.30" @@ -1278,9 +1371,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -1328,14 +1421,17 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.11", "allocator-api2", ] @@ -1487,7 +1583,7 @@ dependencies = [ "http 0.2.12", "hyper", "log", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", @@ -1533,10 +1629,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itertools" version = "0.12.1" @@ -1560,9 +1662,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libmimalloc-sys" @@ -1592,9 +1694,9 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1612,7 +1714,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -1769,7 +1871,10 @@ dependencies = [ name = "nativelink-config" version = "0.3.0" dependencies = [ + "byte-unit", + "humantime", "serde", + "serde_json5", "shellexpand", ] @@ -1813,7 +1918,7 @@ dependencies = [ "async-trait", "blake3", "futures", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lru", "nativelink-config", "nativelink-error", @@ -2018,9 +2123,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -2103,15 +2208,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -2122,9 +2227,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" dependencies = [ "memchr", "thiserror", @@ -2133,9 +2238,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" dependencies = [ "pest", "pest_generator", @@ -2143,9 +2248,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" dependencies = [ "pest", "pest_meta", @@ -2156,9 +2261,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" dependencies = [ "once_cell", "pest", @@ -2167,9 +2272,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -2249,6 +2354,15 @@ dependencies = [ "syn 2.0.52", ] +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2338,9 +2452,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "9554e3ab233f0a932403704f1a1d08c30d5ccd931adfdfa1e8b5a19b52c1d55a" dependencies = [ "anyhow", "itertools", @@ -2358,6 +2472,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quote" version = "1.0.35" @@ -2367,6 +2501,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -2445,6 +2585,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "redox_users" version = "0.4.5" @@ -2508,9 +2657,18 @@ checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "relative-path" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] [[package]] name = "rfc6979" @@ -2538,6 +2696,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "roxmltree" version = "0.14.1" @@ -2547,11 +2734,27 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rust_decimal" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc_version" @@ -2577,9 +2780,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -2641,15 +2844,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" @@ -2674,15 +2877,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" @@ -2709,6 +2912,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "sec1" version = "0.3.0" @@ -2725,11 +2934,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -2738,9 +2947,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -2748,24 +2957,24 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.198" +version = "1.0.201" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.201" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" dependencies = [ "proc-macro2", "quote", @@ -2774,9 +2983,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "indexmap 2.2.6", "itoa", @@ -2888,6 +3097,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "slab" version = "0.4.9" @@ -2905,9 +3120,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2969,12 +3184,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "sync_wrapper" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.10.1" @@ -2989,18 +3222,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", @@ -3120,7 +3353,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio", ] @@ -3149,16 +3382,32 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap 2.2.6", + "toml_datetime", + "winnow", ] [[package]] @@ -3411,6 +3660,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8parse" version = "0.2.1" @@ -3626,6 +3881,24 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -3640,18 +3913,18 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", @@ -3660,6 +3933,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63381fa6624bf92130a6b87c0d07380116f80b565c42cf0d754136f0238359ef" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel index 51097f31e..a04ce4749 100644 --- a/nativelink-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -3,6 +3,7 @@ load( "rust_doc", "rust_doc_test", "rust_library", + "rust_test_suite", ) rust_library( @@ -16,11 +17,29 @@ rust_library( ], visibility = ["//visibility:public"], deps = [ + "@crates//:byte-unit", + "@crates//:humantime", "@crates//:serde", "@crates//:shellexpand", ], ) +rust_test_suite( + name = "integration", + timeout = "short", + srcs = [ + "tests/deserialization_test.rs", + ], + deps = [ + "//nativelink-config", + "//nativelink-error", + "@crates//:byte-unit", + "@crates//:humantime", + "@crates//:serde", + "@crates//:serde_json5", + ], +) + rust_doc( name = "docs", crate = ":nativelink-config", diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml index 2df4dde66..6e4ba8066 100644 --- a/nativelink-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -4,5 +4,8 @@ version = "0.3.0" edition = "2021" [dependencies] +byte-unit = "5.1.4" +humantime = "2.1.0" serde = { version = "1.0.198", features = ["derive"] } +serde_json5 = "0.1.0" shellexpand = "3.1.0" diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index a8461d6da..d46ff0004 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -18,6 +18,7 @@ use serde::Deserialize; use crate::schedulers::SchedulerConfig; use crate::serde_utils::{ + convert_data_size_with_shellexpand, convert_duration_with_shellexpand, convert_numeric_with_shellexpand, convert_optional_numeric_with_shellexpand, convert_optional_string_with_shellexpand, convert_string_with_shellexpand, convert_vec_string_with_shellexpand, @@ -139,7 +140,7 @@ pub struct ByteStreamConfig { /// 16KiB - 64KiB is optimal. /// /// Defaults: 64KiB - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_bytes_per_stream: usize, /// In the event a client disconnects while uploading a blob, we will hold @@ -148,7 +149,7 @@ pub struct ByteStreamConfig { /// the same blob. /// /// Defaults: 10 (seconds) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub persist_stream_on_disconnect_timeout: usize, } @@ -557,7 +558,7 @@ pub struct LocalWorkerConfig { /// longer than this time limit, the task will be rejected. Value in seconds. /// /// Default: 1200 (seconds / 20 mins) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub max_action_timeout: usize, /// If timeout is handled in `entrypoint` or another wrapper script. @@ -667,7 +668,7 @@ pub struct GlobalConfig { /// a new file descriptor because the limit has been reached. /// /// Default: 1000 (1 second) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub idle_file_descriptor_timeout_millis: u64, /// This flag can be used to prevent metrics from being collected at runtime. @@ -695,7 +696,7 @@ pub struct GlobalConfig { /// digest. /// /// Default: 1024*1024 (1MiB) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub default_digest_size_health_check: usize, } diff --git a/nativelink-config/src/lib.rs b/nativelink-config/src/lib.rs index b518c5ef1..866d9a9e6 100644 --- a/nativelink-config/src/lib.rs +++ b/nativelink-config/src/lib.rs @@ -14,5 +14,5 @@ pub mod cas_server; pub mod schedulers; -mod serde_utils; +pub mod serde_utils; pub mod stores; diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index ddd522baf..76f475734 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use serde::Deserialize; -use crate::serde_utils::convert_numeric_with_shellexpand; +use crate::serde_utils::{convert_duration_with_shellexpand, convert_numeric_with_shellexpand}; use crate::stores::{GrpcEndpoint, Retry, StoreRefName}; #[allow(non_camel_case_types)] @@ -97,13 +97,13 @@ pub struct SimpleScheduler { /// The amount of time to retain completed actions in memory for in case /// a WaitExecution is called after the action has completed. /// Default: 60 (seconds) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub retain_completed_for_s: u64, /// Remove workers from pool once the worker has not responded in this /// amount of time in seconds. /// Default: 5 (seconds) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub worker_timeout_s: u64, /// If a job returns an internal error or times out this many times when diff --git a/nativelink-config/src/serde_utils.rs b/nativelink-config/src/serde_utils.rs index 8fead477c..9d754e3dd 100644 --- a/nativelink-config/src/serde_utils.rs +++ b/nativelink-config/src/serde_utils.rs @@ -16,6 +16,8 @@ use std::fmt; use std::marker::PhantomData; use std::str::FromStr; +use byte_unit::Byte; +use humantime::parse_duration; use serde::{de, Deserialize, Deserializer}; /// Helper for serde macro so you can use shellexpand variables in the json configuration @@ -138,3 +140,81 @@ pub fn convert_optional_string_with_shellexpand<'de, D: Deserializer<'de>>( Ok(None) } } + +pub fn convert_data_size_with_shellexpand<'de, D, T, E>(deserializer: D) -> Result +where + D: Deserializer<'de>, + E: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, +{ + // define a visitor that deserializes + // `ActualData` encoded as json within a string + struct USizeVisitor>(PhantomData); + + impl<'de, T, FromStrErr> de::Visitor<'de> for USizeVisitor + where + FromStrErr: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, + { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string containing json data") + } + + fn visit_i64(self, v: i64) -> Result { + v.try_into().map_err(de::Error::custom) + } + + fn visit_str(self, v: &str) -> Result { + let expanded = (*shellexpand::env(v).map_err(de::Error::custom)?).to_string(); + let byte_size = Byte::parse_str(expanded, true).map_err(de::Error::custom)?; + let byte_size_u128 = byte_size.as_u128(); + T::try_from(byte_size_u128.try_into().map_err(de::Error::custom)?) + .map_err(de::Error::custom) + } + } + + deserializer.deserialize_any(USizeVisitor::(PhantomData:: {})) +} + +pub fn convert_duration_with_shellexpand<'de, D, T, E>(deserializer: D) -> Result +where + D: Deserializer<'de>, + E: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, +{ + // define a visitor that deserializes + // `ActualData` encoded as json within a string + struct USizeVisitor>(PhantomData); + + impl<'de, T, FromStrErr> de::Visitor<'de> for USizeVisitor + where + FromStrErr: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, + { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string containing json data") + } + + fn visit_i64(self, v: i64) -> Result { + v.try_into().map_err(de::Error::custom) + } + + fn visit_str(self, v: &str) -> Result { + let expanded = (*shellexpand::env(v).map_err(de::Error::custom)?).to_string(); + let duration = parse_duration(&expanded).map_err(de::Error::custom)?; + let duration_secs = duration.as_secs(); + T::try_from(duration_secs.try_into().map_err(de::Error::custom)?) + .map_err(de::Error::custom) + } + } + + deserializer.deserialize_any(USizeVisitor::(PhantomData:: {})) +} diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index 68d3f7553..3134d5eee 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; use crate::serde_utils::{ + convert_data_size_with_shellexpand, convert_duration_with_shellexpand, convert_numeric_with_shellexpand, convert_optional_string_with_shellexpand, convert_string_with_shellexpand, convert_vec_string_with_shellexpand, }; @@ -203,7 +204,7 @@ pub struct ShardStore { #[serde(deny_unknown_fields)] pub struct SizePartitioningStore { /// Size to partition the data on. - #[serde(deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(deserialize_with = "convert_data_size_with_shellexpand")] pub size: u64, /// Store to send data when object is < (less than) size. @@ -243,7 +244,7 @@ pub struct FilesystemStore { /// Buffer size to use when reading files. Generally this should be left /// to the default value except for testing. /// Default: 32k. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub read_buffer_size: u32, /// Policy used to evict items out of the store. Failure to set this @@ -255,7 +256,7 @@ pub struct FilesystemStore { /// value is used to determine an entry's actual size on disk consumed /// For a 4KB block size filesystem, a 1B file actually consumes 4KB /// Default: 4096 - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub block_size: u64, } @@ -297,7 +298,7 @@ pub struct DedupStore { /// deciding where to partition the data. /// /// Default: 65536 (64k) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub min_size: u32, /// A best-effort attempt will be made to keep the average size @@ -311,13 +312,13 @@ pub struct DedupStore { /// details. /// /// Default: 262144 (256k) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub normal_size: u32, /// Maximum size a chunk is allowed to be. /// /// Default: 524288 (512k) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_size: u32, /// Due to implementation detail, we want to prefer to download @@ -396,7 +397,7 @@ pub struct Lz4Config { /// compression ratios. /// /// Default: 65536 (64k). - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub block_size: u32, /// Maximum size allowed to attempt to deserialize data into. @@ -407,7 +408,7 @@ pub struct Lz4Config { /// allow you to specify the maximum that we'll attempt deserialize. /// /// Default: value in `block_size`. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_decode_block_size: u32, } @@ -447,19 +448,19 @@ pub struct CompressionStore { pub struct EvictionPolicy { /// Maximum number of bytes before eviction takes place. /// Default: 0. Zero means never evict based on size. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_bytes: usize, /// When eviction starts based on hitting max_bytes, continue until /// max_bytes - evict_bytes is met to create a low watermark. This stops /// operations from thrashing when the store is close to the limit. /// Default: 0 - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub evict_bytes: usize, /// Maximum number of seconds for an entry to live before an eviction. /// Default: 0. Zero means never evict based on time. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub max_seconds: u32, /// Maximum size of the store before an eviction takes place. diff --git a/nativelink-config/tests/deserialization_test.rs b/nativelink-config/tests/deserialization_test.rs new file mode 100644 index 000000000..2951676fc --- /dev/null +++ b/nativelink-config/tests/deserialization_test.rs @@ -0,0 +1,71 @@ +// Copyright 2023 The NativeLink Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use nativelink_config::serde_utils::{ + convert_data_size_with_shellexpand, convert_duration_with_shellexpand, +}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct DurationEntity { + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + duration: usize, +} + +#[derive(Deserialize)] +struct DataSizeEntity { + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + data_size: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_duration_human_readable_deserialize() { + let example = r#" + {"duration": "1m 10s"} + "#; + let deserialized: DurationEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.duration, 70); + } + + #[test] + fn test_duration_usize_deserialize() { + let example = r#" + {"duration": 10} + "#; + let deserialized: DurationEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.duration, 10); + } + + #[test] + fn test_data_size_unit_deserialize() { + let example = r#" + {"data_size": "1KiB"} + "#; + let deserialized: DataSizeEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.data_size, 1024); + } + + #[test] + fn test_data_size_usize_deserialize() { + let example = r#" + {"data_size": 10} + "#; + let deserialized: DataSizeEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.data_size, 10); + } +}