From c963b67d2d693fda21e1ecf08bce1c8b0445d20b Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Thu, 25 Jan 2024 11:20:05 -0800 Subject: [PATCH] Introduce component based health Introduce a component based health check system. Each type of component should be able to opt into registering handlers implement some mechanical checks of health. Health in this context is functionality expected to work but runtime wise are semi no-op in terms of influencing the underlying storage / rpc systems. Opting in to the system requires for component to define a `HealthStatusIndicator` and implement the `check_health()` function. Registration should automatically be done by the existence of implementation and calling the `Store.register_health()` function in component implementation. At the moment only `Store` based components can register and a single health check is defined for `FilesystemStore`. The `/status` endpoint has been updated to return the resulting string serialized instances of `HealthStatus`, smart serialization of such objects is not implemented at this time. --- Cargo.Bazel.lock | 731 ++++++++++-------- Cargo.lock | 13 + nativelink-service/tests/ac_server_test.rs | 2 + .../tests/bytestream_server_test.rs | 1 + nativelink-service/tests/cas_server_test.rs | 1 + nativelink-store/BUILD.bazel | 1 + nativelink-store/Cargo.toml | 1 + nativelink-store/src/default_store_factory.rs | 31 +- nativelink-store/src/filesystem_store.rs | 69 +- nativelink-util/BUILD.bazel | 2 + nativelink-util/Cargo.toml | 1 + nativelink-util/src/health_utils.rs | 144 ++++ nativelink-util/src/lib.rs | 1 + nativelink-util/src/store_trait.rs | 3 + nativelink-util/tests/health_utils_test.rs | 179 +++++ src/bin/nativelink.rs | 53 +- 16 files changed, 872 insertions(+), 361 deletions(-) create mode 100644 nativelink-util/src/health_utils.rs create mode 100644 nativelink-util/tests/health_utils_test.rs diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index ab628e588e..deffd8f54a 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "af852655d9eacd497a80742a4a6e9066b95c0a236ee1b77f438a7a8d7fef1237", + "checksum": "a2ea30b5ef14d653d808be96bbcb38b1786ff546eb1a8a4973d4de1d604ecf6a", "crates": { "addr2line 0.21.0": { "name": "addr2line", @@ -174,7 +174,6 @@ ], "crate_features": { "common": [ - "default", "perf-literal", "std" ], @@ -230,13 +229,13 @@ }, "license": "MIT OR Apache-2.0" }, - "anstream 0.6.5": { + "anstream 0.6.11": { "name": "anstream", - "version": "0.6.5", + "version": "0.6.11", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/anstream/0.6.5/download", - "sha256": "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" + "url": "https://crates.io/api/v1/crates/anstream/0.6.11/download", + "sha256": "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" } }, "targets": [ @@ -296,7 +295,7 @@ } }, "edition": "2021", - "version": "0.6.5" + "version": "0.6.11" }, "license": "MIT OR Apache-2.0" }, @@ -618,11 +617,11 @@ "deps": { "common": [ { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "serde_json" } ], @@ -642,13 +641,13 @@ }, "license": "MIT" }, - "async-lock 3.2.0": { + "async-lock 3.3.0": { "name": "async-lock", - "version": "3.2.0", + "version": "3.3.0", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/async-lock/3.2.0/download", - "sha256": "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" + "url": "https://crates.io/api/v1/crates/async-lock/3.3.0/download", + "sha256": "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" } }, "targets": [ @@ -677,7 +676,7 @@ "deps": { "common": [ { - "id": "event-listener 4.0.2", + "id": "event-listener 4.0.3", "target": "event_listener" }, { @@ -691,11 +690,58 @@ ], "selects": {} }, - "edition": "2018", - "version": "3.2.0" + "edition": "2021", + "version": "3.3.0" }, "license": "Apache-2.0 OR MIT" }, + "async-recursion 1.0.5": { + "name": "async-recursion", + "version": "1.0.5", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/async-recursion/1.0.5/download", + "sha256": "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "async_recursion", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "async_recursion", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.78", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.35", + "target": "quote" + }, + { + "id": "syn 2.0.48", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.5" + }, + "license": "MIT OR Apache-2.0" + }, "async-stream 0.3.5": { "name": "async-stream", "version": "0.3.5", @@ -776,7 +822,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -784,7 +830,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -836,7 +882,7 @@ "target": "build_script_build" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -844,7 +890,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -1237,7 +1283,7 @@ "target": "tracing" }, { - "id": "uuid 1.6.1", + "id": "uuid 1.7.0", "target": "uuid" } ], @@ -1361,7 +1407,7 @@ "target": "percent_encoding" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { @@ -1456,7 +1502,7 @@ "target": "http" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { @@ -1547,7 +1593,7 @@ "target": "http" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { @@ -1642,7 +1688,7 @@ "target": "http" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { @@ -1747,7 +1793,7 @@ "target": "percent_encoding" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { @@ -2142,7 +2188,7 @@ "target": "pretty_assertions" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { @@ -2150,7 +2196,7 @@ "target": "roxmltree" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "serde_json" }, { @@ -2308,11 +2354,11 @@ "target": "rustls" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "serde_json" }, { @@ -2508,7 +2554,7 @@ "selects": { "cfg(aws_sdk_unstable)": [ { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" } ] @@ -2751,11 +2797,11 @@ "target": "pin_project_lite" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "serde_json" }, { @@ -2972,7 +3018,7 @@ "target": "addr2line" }, { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" }, { @@ -3041,13 +3087,13 @@ }, "license": "Apache-2.0 OR MIT" }, - "base64 0.21.5": { + "base64 0.21.7": { "name": "base64", - "version": "0.21.5", + "version": "0.21.7", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/base64/0.21.5/download", - "sha256": "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + "url": "https://crates.io/api/v1/crates/base64/0.21.7/download", + "sha256": "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" } }, "targets": [ @@ -3075,7 +3121,7 @@ "selects": {} }, "edition": "2018", - "version": "0.21.5" + "version": "0.21.7" }, "license": "MIT OR Apache-2.0" }, @@ -3195,7 +3241,7 @@ "deps": { "common": [ { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" } ], @@ -3242,13 +3288,13 @@ }, "license": "MIT/Apache-2.0" }, - "bitflags 2.4.1": { + "bitflags 2.4.2": { "name": "bitflags", - "version": "2.4.1", + "version": "2.4.2", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/bitflags/2.4.1/download", - "sha256": "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + "url": "https://crates.io/api/v1/crates/bitflags/2.4.2/download", + "sha256": "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" } }, "targets": [ @@ -3274,7 +3320,7 @@ "selects": {} }, "edition": "2021", - "version": "2.4.1" + "version": "2.4.2" }, "license": "MIT OR Apache-2.0" }, @@ -3556,7 +3602,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ] @@ -3597,13 +3643,13 @@ }, "license": "MIT/Apache-2.0" }, - "clap 4.4.12": { + "clap 4.4.18": { "name": "clap", - "version": "4.4.12", + "version": "4.4.18", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/clap/4.4.12/download", - "sha256": "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" + "url": "https://crates.io/api/v1/crates/clap/4.4.18/download", + "sha256": "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" } }, "targets": [ @@ -3638,7 +3684,7 @@ "deps": { "common": [ { - "id": "clap_builder 4.4.12", + "id": "clap_builder 4.4.18", "target": "clap_builder" } ], @@ -3654,17 +3700,17 @@ ], "selects": {} }, - "version": "4.4.12" + "version": "4.4.18" }, "license": "MIT OR Apache-2.0" }, - "clap_builder 4.4.12": { + "clap_builder 4.4.18": { "name": "clap_builder", - "version": "4.4.12", + "version": "4.4.18", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/clap_builder/4.4.12/download", - "sha256": "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" + "url": "https://crates.io/api/v1/crates/clap_builder/4.4.18/download", + "sha256": "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" } }, "targets": [ @@ -3697,7 +3743,7 @@ "deps": { "common": [ { - "id": "anstream 0.6.5", + "id": "anstream 0.6.11", "target": "anstream" }, { @@ -3716,7 +3762,7 @@ "selects": {} }, "edition": "2021", - "version": "4.4.12" + "version": "4.4.18" }, "license": "MIT OR Apache-2.0" }, @@ -3758,7 +3804,7 @@ "target": "heck" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -3766,7 +3812,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -3871,7 +3917,7 @@ "deps": { "common": [ { - "id": "crossbeam-utils 0.8.18", + "id": "crossbeam-utils 0.8.19", "target": "crossbeam_utils" } ], @@ -3982,11 +4028,11 @@ "target": "console_api" }, { - "id": "crossbeam-channel 0.5.10", + "id": "crossbeam-channel 0.5.11", "target": "crossbeam_channel" }, { - "id": "crossbeam-utils 0.8.18", + "id": "crossbeam-utils 0.8.19", "target": "crossbeam_utils" }, { @@ -4006,11 +4052,11 @@ "target": "prost_types" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "serde_json" }, { @@ -4148,7 +4194,7 @@ "target": "core_foundation_sys" }, { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -4196,13 +4242,13 @@ }, "license": "MIT OR Apache-2.0" }, - "cpufeatures 0.2.11": { + "cpufeatures 0.2.12": { "name": "cpufeatures", - "version": "0.2.11", + "version": "0.2.12", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/cpufeatures/0.2.11/download", - "sha256": "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" + "url": "https://crates.io/api/v1/crates/cpufeatures/0.2.12/download", + "sha256": "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" } }, "targets": [ @@ -4226,32 +4272,32 @@ "selects": { "aarch64-linux-android": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], "cfg(all(target_arch = \"aarch64\", target_os = \"linux\"))": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], "cfg(all(target_arch = \"aarch64\", target_vendor = \"apple\"))": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], "cfg(all(target_arch = \"loongarch64\", target_os = \"linux\"))": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ] } }, "edition": "2018", - "version": "0.2.11" + "version": "0.2.12" }, "license": "MIT OR Apache-2.0" }, @@ -4381,13 +4427,13 @@ }, "license": "MIT OR Apache-2.0" }, - "crossbeam-channel 0.5.10": { + "crossbeam-channel 0.5.11": { "name": "crossbeam-channel", - "version": "0.5.10", + "version": "0.5.11", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/crossbeam-channel/0.5.10/download", - "sha256": "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" + "url": "https://crates.io/api/v1/crates/crossbeam-channel/0.5.11/download", + "sha256": "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" } }, "targets": [ @@ -4416,28 +4462,24 @@ "deps": { "common": [ { - "id": "cfg-if 1.0.0", - "target": "cfg_if" - }, - { - "id": "crossbeam-utils 0.8.18", + "id": "crossbeam-utils 0.8.19", "target": "crossbeam_utils" } ], "selects": {} }, "edition": "2021", - "version": "0.5.10" + "version": "0.5.11" }, "license": "MIT OR Apache-2.0" }, - "crossbeam-utils 0.8.18": { + "crossbeam-utils 0.8.19": { "name": "crossbeam-utils", - "version": "0.8.18", + "version": "0.8.19", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/crossbeam-utils/0.8.18/download", - "sha256": "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" + "url": "https://crates.io/api/v1/crates/crossbeam-utils/0.8.19/download", + "sha256": "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" } }, "targets": [ @@ -4475,18 +4517,14 @@ "deps": { "common": [ { - "id": "cfg-if 1.0.0", - "target": "cfg_if" - }, - { - "id": "crossbeam-utils 0.8.18", + "id": "crossbeam-utils 0.8.19", "target": "build_script_build" } ], "selects": {} }, "edition": "2021", - "version": "0.8.18" + "version": "0.8.19" }, "build_script_attrs": { "data_glob": [ @@ -4871,7 +4909,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -5184,19 +5222,19 @@ "selects": { "cfg(target_os = \"hermit\")": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], "cfg(target_os = \"wasi\")": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -5213,13 +5251,13 @@ }, "license": "MIT OR Apache-2.0" }, - "event-listener 4.0.2": { + "event-listener 4.0.3": { "name": "event-listener", - "version": "4.0.2", + "version": "4.0.3", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/event-listener/4.0.2/download", - "sha256": "218a870470cce1469024e9fb66b901aa983929d81304a1cdb299f28118e550d5" + "url": "https://crates.io/api/v1/crates/event-listener/4.0.3/download", + "sha256": "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" } }, "targets": [ @@ -5266,7 +5304,7 @@ } }, "edition": "2021", - "version": "4.0.2" + "version": "4.0.3" }, "license": "Apache-2.0 OR MIT" }, @@ -5304,7 +5342,7 @@ "deps": { "common": [ { - "id": "event-listener 4.0.2", + "id": "event-listener 4.0.3", "target": "event_listener" }, { @@ -5351,7 +5389,7 @@ "target": "proc_macro_error" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -5492,7 +5530,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -5987,7 +6025,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -5995,7 +6033,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -6251,13 +6289,13 @@ }, "license": "MIT" }, - "getrandom 0.2.11": { + "getrandom 0.2.12": { "name": "getrandom", - "version": "0.2.11", + "version": "0.2.12", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/getrandom/0.2.11/download", - "sha256": "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" + "url": "https://crates.io/api/v1/crates/getrandom/0.2.12/download", + "sha256": "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" } }, "targets": [ @@ -6298,14 +6336,14 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ] } }, "edition": "2018", - "version": "0.2.11" + "version": "0.2.12" }, "license": "MIT OR Apache-2.0" }, @@ -6386,13 +6424,13 @@ }, "license": "MIT/Apache-2.0" }, - "h2 0.3.22": { + "h2 0.3.24": { "name": "h2", - "version": "0.3.22", + "version": "0.3.24", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/h2/0.3.22/download", - "sha256": "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" + "url": "https://crates.io/api/v1/crates/h2/0.3.24/download", + "sha256": "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" } }, "targets": [ @@ -6461,7 +6499,7 @@ "selects": {} }, "edition": "2018", - "version": "0.3.22" + "version": "0.3.24" }, "license": "MIT" }, @@ -6591,7 +6629,7 @@ "deps": { "common": [ { - "id": "base64 0.21.5", + "id": "base64 0.21.7", "target": "base64" }, { @@ -6654,13 +6692,13 @@ }, "license": "MIT OR Apache-2.0" }, - "hermit-abi 0.3.3": { + "hermit-abi 0.3.4": { "name": "hermit-abi", - "version": "0.3.3", + "version": "0.3.4", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/hermit-abi/0.3.3/download", - "sha256": "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + "url": "https://crates.io/api/v1/crates/hermit-abi/0.3.4/download", + "sha256": "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" } }, "targets": [ @@ -6680,7 +6718,7 @@ "**" ], "edition": "2021", - "version": "0.3.3" + "version": "0.3.4" }, "license": "MIT OR Apache-2.0" }, @@ -7082,7 +7120,7 @@ "target": "futures_util" }, { - "id": "h2 0.3.22", + "id": "h2 0.3.24", "target": "h2" }, { @@ -7312,7 +7350,7 @@ "deps": { "common": [ { - "id": "unicode-bidi 0.3.14", + "id": "unicode-bidi 0.3.15", "target": "unicode_bidi" }, { @@ -7548,13 +7586,13 @@ }, "license": "MIT/Apache-2.0" }, - "libc 0.2.151": { + "libc 0.2.152": { "name": "libc", - "version": "0.2.151", + "version": "0.2.152", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/libc/0.2.151/download", - "sha256": "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + "url": "https://crates.io/api/v1/crates/libc/0.2.152/download", + "sha256": "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" } }, "targets": [ @@ -7616,14 +7654,14 @@ "deps": { "common": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "build_script_build" } ], "selects": {} }, "edition": "2015", - "version": "0.2.151" + "version": "0.2.152" }, "build_script_attrs": { "data_glob": [ @@ -7660,11 +7698,11 @@ "deps": { "common": [ { - "id": "bitflags 2.4.1", + "id": "bitflags 2.4.2", "target": "bitflags" }, { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" }, { @@ -7679,13 +7717,13 @@ }, "license": "MIT" }, - "linux-raw-sys 0.4.12": { + "linux-raw-sys 0.4.13": { "name": "linux-raw-sys", - "version": "0.4.12", + "version": "0.4.13", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/linux-raw-sys/0.4.12/download", - "sha256": "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + "url": "https://crates.io/api/v1/crates/linux-raw-sys/0.4.13/download", + "sha256": "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" } }, "targets": [ @@ -7715,7 +7753,7 @@ "selects": {} }, "edition": "2021", - "version": "0.4.12" + "version": "0.4.13" }, "license": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" }, @@ -7874,13 +7912,13 @@ }, "license": "MIT" }, - "lz4_flex 0.11.1": { + "lz4_flex 0.11.2": { "name": "lz4_flex", - "version": "0.11.1", + "version": "0.11.2", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/lz4_flex/0.11.1/download", - "sha256": "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" + "url": "https://crates.io/api/v1/crates/lz4_flex/0.11.2/download", + "sha256": "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" } }, "targets": [ @@ -7919,7 +7957,7 @@ "selects": {} }, "edition": "2021", - "version": "0.11.1" + "version": "0.11.2" }, "license": "MIT" }, @@ -8146,7 +8184,7 @@ "selects": { "cfg(any(target_os = \"linux\", target_os = \"android\", target_os = \"macos\", target_os = \"ios\"))": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -8312,7 +8350,7 @@ "selects": { "cfg(target_os = \"wasi\")": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" }, { @@ -8322,7 +8360,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -8418,7 +8456,7 @@ "deps": { "common": [ { - "id": "async-lock 3.2.0", + "id": "async-lock 3.3.0", "target": "async_lock" }, { @@ -8426,7 +8464,7 @@ "target": "axum" }, { - "id": "clap 4.4.12", + "id": "clap 4.4.18", "target": "clap" }, { @@ -8516,7 +8554,7 @@ "deps": { "common": [ { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { @@ -8707,7 +8745,7 @@ "target": "tracing" }, { - "id": "uuid 1.6.1", + "id": "uuid 1.7.0", "target": "uuid" } ], @@ -8799,7 +8837,7 @@ "target": "tracing" }, { - "id": "uuid 1.6.1", + "id": "uuid 1.7.0", "target": "uuid" } ], @@ -8858,7 +8896,7 @@ "deps": { "common": [ { - "id": "async-lock 3.2.0", + "id": "async-lock 3.3.0", "target": "async_lock" }, { @@ -8910,7 +8948,7 @@ "target": "hyper_rustls" }, { - "id": "lz4_flex 0.11.1", + "id": "lz4_flex 0.11.2", "target": "lz4_flex" }, { @@ -8926,7 +8964,7 @@ "target": "rand" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { @@ -8937,6 +8975,10 @@ "id": "shellexpand 3.1.0", "target": "shellexpand" }, + { + "id": "tempfile 3.9.0", + "target": "tempfile" + }, { "id": "tokio 1.35.1", "target": "tokio" @@ -8958,7 +9000,7 @@ "target": "tracing" }, { - "id": "uuid 1.6.1", + "id": "uuid 1.7.0", "target": "uuid" } ], @@ -9026,7 +9068,7 @@ "deps": { "common": [ { - "id": "async-lock 3.2.0", + "id": "async-lock 3.3.0", "target": "async_lock" }, { @@ -9074,7 +9116,7 @@ "target": "prost_types" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { @@ -9120,6 +9162,10 @@ "edition": "2021", "proc_macro_deps": { "common": [ + { + "id": "async-recursion 1.0.5", + "target": "async_recursion" + }, { "id": "async-trait 0.1.77", "target": "async_trait" @@ -9154,7 +9200,7 @@ "deps": { "common": [ { - "id": "async-lock 3.2.0", + "id": "async-lock 3.3.0", "target": "async_lock" }, { @@ -9194,7 +9240,7 @@ "target": "scopeguard" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { @@ -9202,7 +9248,7 @@ "target": "serde_json5" }, { - "id": "shlex 1.2.0", + "id": "shlex 1.3.0", "target": "shlex" }, { @@ -9222,7 +9268,7 @@ "target": "tracing" }, { - "id": "uuid 1.6.1", + "id": "uuid 1.7.0", "target": "uuid" } ], @@ -9615,13 +9661,13 @@ "selects": { "cfg(not(windows))": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], "cfg(target_os = \"hermit\")": [ { - "id": "hermit-abi 0.3.3", + "id": "hermit-abi 0.3.4", "target": "hermit_abi" } ] @@ -10016,7 +10062,7 @@ "target": "build_script_build" }, { - "id": "smallvec 1.11.2", + "id": "smallvec 1.13.1", "target": "smallvec" } ], @@ -10029,7 +10075,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -10089,13 +10135,13 @@ }, "license": "MIT OR Apache-2.0" }, - "pest 2.7.5": { + "pest 2.7.6": { "name": "pest", - "version": "2.7.5", + "version": "2.7.6", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/pest/2.7.5/download", - "sha256": "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" + "url": "https://crates.io/api/v1/crates/pest/2.7.6/download", + "sha256": "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" } }, "targets": [ @@ -10140,17 +10186,17 @@ "selects": {} }, "edition": "2021", - "version": "2.7.5" + "version": "2.7.6" }, "license": "MIT OR Apache-2.0" }, - "pest_derive 2.7.5": { + "pest_derive 2.7.6": { "name": "pest_derive", - "version": "2.7.5", + "version": "2.7.6", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/pest_derive/2.7.5/download", - "sha256": "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" + "url": "https://crates.io/api/v1/crates/pest_derive/2.7.6/download", + "sha256": "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" } }, "targets": [ @@ -10179,28 +10225,28 @@ "deps": { "common": [ { - "id": "pest 2.7.5", + "id": "pest 2.7.6", "target": "pest" }, { - "id": "pest_generator 2.7.5", + "id": "pest_generator 2.7.6", "target": "pest_generator" } ], "selects": {} }, "edition": "2021", - "version": "2.7.5" + "version": "2.7.6" }, "license": "MIT OR Apache-2.0" }, - "pest_generator 2.7.5": { + "pest_generator 2.7.6": { "name": "pest_generator", - "version": "2.7.5", + "version": "2.7.6", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/pest_generator/2.7.5/download", - "sha256": "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" + "url": "https://crates.io/api/v1/crates/pest_generator/2.7.6/download", + "sha256": "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" } }, "targets": [ @@ -10228,15 +10274,15 @@ "deps": { "common": [ { - "id": "pest 2.7.5", + "id": "pest 2.7.6", "target": "pest" }, { - "id": "pest_meta 2.7.5", + "id": "pest_meta 2.7.6", "target": "pest_meta" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -10244,24 +10290,24 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], "selects": {} }, "edition": "2021", - "version": "2.7.5" + "version": "2.7.6" }, "license": "MIT OR Apache-2.0" }, - "pest_meta 2.7.5": { + "pest_meta 2.7.6": { "name": "pest_meta", - "version": "2.7.5", + "version": "2.7.6", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/pest_meta/2.7.5/download", - "sha256": "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" + "url": "https://crates.io/api/v1/crates/pest_meta/2.7.6/download", + "sha256": "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" } }, "targets": [ @@ -10293,14 +10339,14 @@ "target": "once_cell" }, { - "id": "pest 2.7.5", + "id": "pest 2.7.6", "target": "pest" } ], "selects": {} }, "edition": "2021", - "version": "2.7.5" + "version": "2.7.6" }, "license": "MIT OR Apache-2.0" }, @@ -10414,7 +10460,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -10422,7 +10468,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -10700,11 +10746,11 @@ "target": "build_script_build" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -10770,7 +10816,7 @@ "target": "build_script_build" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -10853,7 +10899,7 @@ "target": "build_script_build" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -10882,13 +10928,13 @@ }, "license": "MIT OR Apache-2.0" }, - "proc-macro2 1.0.74": { + "proc-macro2 1.0.78": { "name": "proc-macro2", - "version": "1.0.74", + "version": "1.0.78", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/proc-macro2/1.0.74/download", - "sha256": "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" + "url": "https://crates.io/api/v1/crates/proc-macro2/1.0.78/download", + "sha256": "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" } }, "targets": [ @@ -10926,7 +10972,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "build_script_build" }, { @@ -10937,7 +10983,7 @@ "selects": {} }, "edition": "2021", - "version": "1.0.74" + "version": "1.0.78" }, "build_script_attrs": { "data_glob": [ @@ -11054,7 +11100,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -11062,7 +11108,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -11206,11 +11252,11 @@ "target": "prost_types" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" }, { @@ -11265,7 +11311,7 @@ "target": "itertools" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -11273,7 +11319,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -11365,7 +11411,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" } ], @@ -11428,7 +11474,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ] @@ -11524,7 +11570,7 @@ "deps": { "common": [ { - "id": "getrandom 0.2.11", + "id": "getrandom 0.2.12", "target": "getrandom" } ], @@ -11602,7 +11648,7 @@ "deps": { "common": [ { - "id": "getrandom 0.2.11", + "id": "getrandom 0.2.12", "target": "getrandom" }, { @@ -11621,13 +11667,13 @@ }, "license": "MIT" }, - "regex 1.10.2": { + "regex 1.10.3": { "name": "regex", - "version": "1.10.2", + "version": "1.10.3", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/regex/1.10.2/download", - "sha256": "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" + "url": "https://crates.io/api/v1/crates/regex/1.10.3/download", + "sha256": "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" } }, "targets": [ @@ -11679,7 +11725,7 @@ "target": "memchr" }, { - "id": "regex-automata 0.4.3", + "id": "regex-automata 0.4.4", "target": "regex_automata" }, { @@ -11690,7 +11736,7 @@ "selects": {} }, "edition": "2021", - "version": "1.10.2" + "version": "1.10.3" }, "license": "MIT OR Apache-2.0" }, @@ -11741,13 +11787,13 @@ }, "license": "Unlicense/MIT" }, - "regex-automata 0.4.3": { + "regex-automata 0.4.4": { "name": "regex-automata", - "version": "0.4.3", + "version": "0.4.4", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/regex-automata/0.4.3/download", - "sha256": "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" + "url": "https://crates.io/api/v1/crates/regex-automata/0.4.4/download", + "sha256": "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" } }, "targets": [ @@ -11811,7 +11857,7 @@ "selects": {} }, "edition": "2021", - "version": "0.4.3" + "version": "0.4.4" }, "license": "MIT OR Apache-2.0" }, @@ -12032,7 +12078,7 @@ "deps": { "common": [ { - "id": "getrandom 0.2.11", + "id": "getrandom 0.2.12", "target": "getrandom" }, { @@ -12047,7 +12093,7 @@ "selects": { "cfg(all(any(target_os = \"android\", target_os = \"linux\"), any(target_arch = \"aarch64\", target_arch = \"arm\")))": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -12200,13 +12246,13 @@ }, "license": "MIT/Apache-2.0" }, - "rustix 0.38.28": { + "rustix 0.38.30": { "name": "rustix", - "version": "0.38.28", + "version": "0.38.30", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/rustix/0.38.28/download", - "sha256": "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" + "url": "https://crates.io/api/v1/crates/rustix/0.38.30/download", + "sha256": "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" } }, "targets": [ @@ -12270,24 +12316,24 @@ "deps": { "common": [ { - "id": "bitflags 2.4.1", + "id": "bitflags 2.4.2", "target": "bitflags" }, { - "id": "rustix 0.38.28", + "id": "rustix 0.38.30", "target": "build_script_build" } ], "selects": { "cfg(all(any(target_os = \"android\", target_os = \"linux\"), any(rustix_use_libc, miri, not(all(target_os = \"linux\", target_endian = \"little\", any(target_arch = \"arm\", all(target_arch = \"aarch64\", target_pointer_width = \"64\"), target_arch = \"riscv64\", all(rustix_use_experimental_asm, target_arch = \"powerpc64\"), all(rustix_use_experimental_asm, target_arch = \"mips\"), all(rustix_use_experimental_asm, target_arch = \"mips32r6\"), all(rustix_use_experimental_asm, target_arch = \"mips64\"), all(rustix_use_experimental_asm, target_arch = \"mips64r6\"), target_arch = \"x86\", all(target_arch = \"x86_64\", target_pointer_width = \"64\")))))))": [ { - "id": "linux-raw-sys 0.4.12", + "id": "linux-raw-sys 0.4.13", "target": "linux_raw_sys" } ], "cfg(all(not(rustix_use_libc), not(miri), target_os = \"linux\", target_endian = \"little\", any(target_arch = \"arm\", all(target_arch = \"aarch64\", target_pointer_width = \"64\"), target_arch = \"riscv64\", all(rustix_use_experimental_asm, target_arch = \"powerpc64\"), all(rustix_use_experimental_asm, target_arch = \"mips\"), all(rustix_use_experimental_asm, target_arch = \"mips32r6\"), all(rustix_use_experimental_asm, target_arch = \"mips64\"), all(rustix_use_experimental_asm, target_arch = \"mips64r6\"), target_arch = \"x86\", all(target_arch = \"x86_64\", target_pointer_width = \"64\"))))": [ { - "id": "linux-raw-sys 0.4.12", + "id": "linux-raw-sys 0.4.13", "target": "linux_raw_sys" } ], @@ -12298,7 +12344,7 @@ "alias": "libc_errno" }, { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -12316,7 +12362,7 @@ } }, "edition": "2021", - "version": "0.38.28" + "version": "0.38.30" }, "build_script_attrs": { "data_glob": [ @@ -12412,13 +12458,13 @@ }, "license": "Apache-2.0 OR ISC OR MIT" }, - "rustls 0.22.1": { + "rustls 0.22.2": { "name": "rustls", - "version": "0.22.1", + "version": "0.22.2", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/rustls/0.22.1/download", - "sha256": "fe6b63262c9fcac8659abfaa96cac103d28166d3ff3eaf8f412e19f3ae9e5a48" + "url": "https://crates.io/api/v1/crates/rustls/0.22.2/download", + "sha256": "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" } }, "targets": [ @@ -12466,7 +12512,7 @@ "target": "ring" }, { - "id": "rustls 0.22.1", + "id": "rustls 0.22.2", "target": "build_script_build" }, { @@ -12475,7 +12521,7 @@ "alias": "pki_types" }, { - "id": "rustls-webpki 0.102.0", + "id": "rustls-webpki 0.102.1", "target": "webpki" }, { @@ -12490,7 +12536,7 @@ "selects": {} }, "edition": "2021", - "version": "0.22.1" + "version": "0.22.2" }, "build_script_attrs": { "data_glob": [ @@ -12594,7 +12640,7 @@ "deps": { "common": [ { - "id": "base64 0.21.5", + "id": "base64 0.21.7", "target": "base64" } ], @@ -12640,7 +12686,7 @@ "deps": { "common": [ { - "id": "base64 0.21.5", + "id": "base64 0.21.7", "target": "base64" }, { @@ -12745,13 +12791,13 @@ }, "license": "ISC" }, - "rustls-webpki 0.102.0": { + "rustls-webpki 0.102.1": { "name": "rustls-webpki", - "version": "0.102.0", + "version": "0.102.1", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/rustls-webpki/0.102.0/download", - "sha256": "de2635c8bc2b88d367767c5de8ea1d8db9af3f6219eba28442242d9ab81d1b89" + "url": "https://crates.io/api/v1/crates/rustls-webpki/0.102.1/download", + "sha256": "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" } }, "targets": [ @@ -12797,7 +12843,7 @@ "selects": {} }, "edition": "2021", - "version": "0.102.0" + "version": "0.102.1" }, "license": "ISC" }, @@ -13123,7 +13169,7 @@ "target": "core_foundation_sys" }, { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" }, { @@ -13176,7 +13222,7 @@ "target": "core_foundation_sys" }, { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -13247,13 +13293,13 @@ }, "license": "MIT OR Apache-2.0" }, - "serde 1.0.194": { + "serde 1.0.195": { "name": "serde", - "version": "1.0.194", + "version": "1.0.195", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/serde/1.0.194/download", - "sha256": "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" + "url": "https://crates.io/api/v1/crates/serde/1.0.195/download", + "sha256": "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" } }, "targets": [ @@ -13293,7 +13339,7 @@ "deps": { "common": [ { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "build_script_build" } ], @@ -13303,13 +13349,13 @@ "proc_macro_deps": { "common": [ { - "id": "serde_derive 1.0.194", + "id": "serde_derive 1.0.195", "target": "serde_derive" } ], "selects": {} }, - "version": "1.0.194" + "version": "1.0.195" }, "build_script_attrs": { "data_glob": [ @@ -13318,13 +13364,13 @@ }, "license": "MIT OR Apache-2.0" }, - "serde_derive 1.0.194": { + "serde_derive 1.0.195": { "name": "serde_derive", - "version": "1.0.194", + "version": "1.0.195", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/serde_derive/1.0.194/download", - "sha256": "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" + "url": "https://crates.io/api/v1/crates/serde_derive/1.0.195/download", + "sha256": "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" } }, "targets": [ @@ -13352,7 +13398,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -13360,24 +13406,24 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], "selects": {} }, "edition": "2015", - "version": "1.0.194" + "version": "1.0.195" }, "license": "MIT OR Apache-2.0" }, - "serde_json 1.0.110": { + "serde_json 1.0.111": { "name": "serde_json", - "version": "1.0.110", + "version": "1.0.111", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/serde_json/1.0.110/download", - "sha256": "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" + "url": "https://crates.io/api/v1/crates/serde_json/1.0.111/download", + "sha256": "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" } }, "targets": [ @@ -13424,18 +13470,18 @@ "target": "ryu" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "build_script_build" } ], "selects": {} }, "edition": "2021", - "version": "1.0.110" + "version": "1.0.111" }, "build_script_attrs": { "data_glob": [ @@ -13472,11 +13518,11 @@ "deps": { "common": [ { - "id": "pest 2.7.5", + "id": "pest 2.7.6", "target": "pest" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" } ], @@ -13486,7 +13532,7 @@ "proc_macro_deps": { "common": [ { - "id": "pest_derive 2.7.5", + "id": "pest_derive 2.7.6", "target": "pest_derive" } ], @@ -13528,7 +13574,7 @@ "target": "itoa" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" } ], @@ -13579,7 +13625,7 @@ "target": "ryu" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" } ], @@ -13636,7 +13682,7 @@ "selects": { "cfg(any(target_arch = \"aarch64\", target_arch = \"x86\", target_arch = \"x86_64\"))": [ { - "id": "cpufeatures 0.2.11", + "id": "cpufeatures 0.2.12", "target": "cpufeatures" } ] @@ -13693,7 +13739,7 @@ "selects": { "cfg(any(target_arch = \"aarch64\", target_arch = \"x86_64\", target_arch = \"x86\"))": [ { - "id": "cpufeatures 0.2.11", + "id": "cpufeatures 0.2.12", "target": "cpufeatures" } ] @@ -13791,13 +13837,13 @@ }, "license": "MIT/Apache-2.0" }, - "shlex 1.2.0": { + "shlex 1.3.0": { "name": "shlex", - "version": "1.2.0", + "version": "1.3.0", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/shlex/1.2.0/download", - "sha256": "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + "url": "https://crates.io/api/v1/crates/shlex/1.3.0/download", + "sha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" } }, "targets": [ @@ -13824,7 +13870,7 @@ "selects": {} }, "edition": "2015", - "version": "1.2.0" + "version": "1.3.0" }, "license": "MIT OR Apache-2.0" }, @@ -13856,7 +13902,7 @@ "deps": { "common": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -13990,13 +14036,13 @@ }, "license": "MIT" }, - "smallvec 1.11.2": { + "smallvec 1.13.1": { "name": "smallvec", - "version": "1.11.2", + "version": "1.13.1", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/smallvec/1.11.2/download", - "sha256": "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + "url": "https://crates.io/api/v1/crates/smallvec/1.13.1/download", + "sha256": "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" } }, "targets": [ @@ -14016,7 +14062,7 @@ "**" ], "edition": "2018", - "version": "1.11.2" + "version": "1.13.1" }, "license": "MIT OR Apache-2.0" }, @@ -14056,7 +14102,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" } ], @@ -14306,7 +14352,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -14334,13 +14380,13 @@ }, "license": "MIT OR Apache-2.0" }, - "syn 2.0.46": { + "syn 2.0.48": { "name": "syn", - "version": "2.0.46", + "version": "2.0.48", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/syn/2.0.46/download", - "sha256": "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" + "url": "https://crates.io/api/v1/crates/syn/2.0.48/download", + "sha256": "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" } }, "targets": [ @@ -14377,7 +14423,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -14392,7 +14438,7 @@ "selects": {} }, "edition": "2021", - "version": "2.0.46" + "version": "2.0.48" }, "license": "MIT OR Apache-2.0" }, @@ -14465,7 +14511,7 @@ "selects": { "cfg(any(unix, target_os = \"wasi\"))": [ { - "id": "rustix 0.38.28", + "id": "rustix 0.38.30", "target": "rustix" } ], @@ -14578,7 +14624,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -14586,7 +14632,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -14941,7 +14987,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.151", + "id": "libc 0.2.152", "target": "libc" }, { @@ -15042,7 +15088,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -15050,7 +15096,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -15149,7 +15195,7 @@ "deps": { "common": [ { - "id": "rustls 0.22.1", + "id": "rustls 0.22.2", "target": "rustls" }, { @@ -15347,7 +15393,7 @@ "target": "axum" }, { - "id": "base64 0.21.5", + "id": "base64 0.21.7", "target": "base64" }, { @@ -15359,7 +15405,7 @@ "target": "flate2" }, { - "id": "h2 0.3.22", + "id": "h2 0.3.24", "target": "h2" }, { @@ -15484,7 +15530,7 @@ "target": "prettyplease" }, { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -15496,7 +15542,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -15771,7 +15817,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -15779,7 +15825,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -15926,7 +15972,7 @@ "deps": { "common": [ { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { @@ -16006,15 +16052,15 @@ "target": "once_cell" }, { - "id": "regex 1.10.2", + "id": "regex 1.10.3", "target": "regex" }, { - "id": "serde 1.0.194", + "id": "serde 1.0.195", "target": "serde" }, { - "id": "serde_json 1.0.110", + "id": "serde_json 1.0.111", "target": "serde_json" }, { @@ -16022,7 +16068,7 @@ "target": "sharded_slab" }, { - "id": "smallvec 1.11.2", + "id": "smallvec 1.13.1", "target": "smallvec" }, { @@ -16215,13 +16261,13 @@ }, "license": "MIT OR Apache-2.0" }, - "unicode-bidi 0.3.14": { + "unicode-bidi 0.3.15": { "name": "unicode-bidi", - "version": "0.3.14", + "version": "0.3.15", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/unicode-bidi/0.3.14/download", - "sha256": "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" + "url": "https://crates.io/api/v1/crates/unicode-bidi/0.3.15/download", + "sha256": "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" } }, "targets": [ @@ -16248,7 +16294,7 @@ "selects": {} }, "edition": "2018", - "version": "0.3.14" + "version": "0.3.15" }, "license": "MIT OR Apache-2.0" }, @@ -16476,13 +16522,13 @@ }, "license": "Apache-2.0 OR MIT" }, - "uuid 1.6.1": { + "uuid 1.7.0": { "name": "uuid", - "version": "1.6.1", + "version": "1.7.0", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/uuid/1.6.1/download", - "sha256": "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" + "url": "https://crates.io/api/v1/crates/uuid/1.7.0/download", + "sha256": "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" } }, "targets": [ @@ -16504,7 +16550,6 @@ "crate_features": { "common": [ "default", - "getrandom", "rng", "std", "v4" @@ -16514,14 +16559,14 @@ "deps": { "common": [ { - "id": "getrandom 0.2.11", + "id": "getrandom 0.2.12", "target": "getrandom" } ], "selects": {} }, "edition": "2018", - "version": "1.6.1" + "version": "1.7.0" }, "license": "Apache-2.0 OR MIT" }, @@ -16777,7 +16822,7 @@ "target": "either" }, { - "id": "rustix 0.38.28", + "id": "rustix 0.38.30", "target": "rustix" } ], @@ -18154,7 +18199,7 @@ "deps": { "common": [ { - "id": "proc-macro2 1.0.74", + "id": "proc-macro2 1.0.78", "target": "proc_macro2" }, { @@ -18162,7 +18207,7 @@ "target": "quote" }, { - "id": "syn 2.0.46", + "id": "syn 2.0.48", "target": "syn" } ], @@ -18399,7 +18444,8 @@ ] }, "direct_deps": [ - "async-lock 3.2.0", + "async-lock 3.3.0", + "async-recursion 1.0.5", "async-trait 0.1.77", "aws-config 0.57.2", "aws-sdk-s3 0.35.0", @@ -18409,7 +18455,7 @@ "blake3 1.5.0", "byteorder 1.5.0", "bytes 1.5.0", - "clap 4.4.12", + "clap 4.4.18", "console-subscriber 0.2.0", "filetime 0.2.23", "formatx 0.2.2", @@ -18420,7 +18466,7 @@ "hyper-rustls 0.24.2", "log 0.4.20", "lru 0.12.1", - "lz4_flex 0.11.1", + "lz4_flex 0.11.2", "parking_lot 0.12.1", "pin-project-lite 0.2.13", "prometheus-client 0.21.2", @@ -18430,11 +18476,12 @@ "relative-path 1.9.2", "rustls-pemfile 2.0.0", "scopeguard 1.2.0", - "serde 1.0.194", + "serde 1.0.195", "serde_json5 0.1.0", "sha2 0.10.8", "shellexpand 3.1.0", - "shlex 1.2.0", + "shlex 1.3.0", + "tempfile 3.9.0", "tokio 1.35.1", "tokio-rustls 0.25.0", "tokio-stream 0.1.14", @@ -18443,7 +18490,7 @@ "tower 0.4.13", "tracing 0.1.40", "tracing-subscriber 0.3.18", - "uuid 1.6.1" + "uuid 1.7.0" ], "direct_dev_deps": [ "aws-smithy-types 0.57.2", diff --git a/Cargo.lock b/Cargo.lock index 7b81664116..9634bb5478 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,6 +132,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-recursion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -1791,6 +1802,7 @@ dependencies = [ "serde", "sha2", "shellexpand", + "tempfile", "tokio", "tokio-stream", "tokio-util", @@ -1804,6 +1816,7 @@ name = "nativelink-util" version = "0.2.0" dependencies = [ "async-lock", + "async-recursion", "async-trait", "blake3", "bytes", diff --git a/nativelink-service/tests/ac_server_test.rs b/nativelink-service/tests/ac_server_test.rs index 04a10234e7..08dddd1dcc 100644 --- a/nativelink-service/tests/ac_server_test.rs +++ b/nativelink-service/tests/ac_server_test.rs @@ -55,6 +55,7 @@ async fn make_store_manager() -> Result, Error> { &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), + None, ) .await?, ); @@ -64,6 +65,7 @@ async fn make_store_manager() -> Result, Error> { &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), + None, ) .await?, ); diff --git a/nativelink-service/tests/bytestream_server_test.rs b/nativelink-service/tests/bytestream_server_test.rs index f2573c1042..1a7137778f 100644 --- a/nativelink-service/tests/bytestream_server_test.rs +++ b/nativelink-service/tests/bytestream_server_test.rs @@ -41,6 +41,7 @@ async fn make_store_manager() -> Result, Error> { &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), + None, ) .await?, ); diff --git a/nativelink-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs index f6f636899a..ac41380612 100644 --- a/nativelink-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -41,6 +41,7 @@ async fn make_store_manager() -> Result, Error> { &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), + None, ) .await?, ); diff --git a/nativelink-store/BUILD.bazel b/nativelink-store/BUILD.bazel index 94a4abb453..106bf93480 100644 --- a/nativelink-store/BUILD.bazel +++ b/nativelink-store/BUILD.bazel @@ -58,6 +58,7 @@ rust_library( "@crate_index//:serde", "@crate_index//:sha2", "@crate_index//:shellexpand", + "@crate_index//:tempfile", "@crate_index//:tokio", "@crate_index//:tokio-stream", "@crate_index//:tokio-util", diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml index a1c67bdb17..0ffd051c80 100644 --- a/nativelink-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -30,6 +30,7 @@ rand = "0.8.5" serde = "1.0.193" sha2 = "0.10.8" shellexpand = "3.1.0" +tempfile = "3.9.0" tokio = { version = "1.35.1" } tokio-stream = { version = "0.1.14", features = ["fs"] } tokio-util = { version = "0.7.10" } diff --git a/nativelink-store/src/default_store_factory.rs b/nativelink-store/src/default_store_factory.rs index 6ce68cbf6a..95c3d184fb 100644 --- a/nativelink-store/src/default_store_factory.rs +++ b/nativelink-store/src/default_store_factory.rs @@ -19,6 +19,7 @@ use futures::stream::FuturesOrdered; use futures::{Future, TryStreamExt}; use nativelink_config::stores::StoreConfig; use nativelink_error::Error; +use nativelink_util::health_utils::HealthRegistry; use nativelink_util::metrics_utils::Registry; use nativelink_util::store_trait::Store; @@ -44,6 +45,7 @@ pub fn store_factory<'a>( backend: &'a StoreConfig, store_manager: &'a Arc, maybe_store_metrics: Option<&'a mut Registry>, + maybe_health_registry: Option<&'a mut HealthRegistry>, ) -> Pin> { Box::pin(async move { let store: Arc = match backend { @@ -51,36 +53,36 @@ pub fn store_factory<'a>( StoreConfig::experimental_s3_store(config) => Arc::new(S3Store::new(config).await?), StoreConfig::verify(config) => Arc::new(VerifyStore::new( config, - store_factory(&config.backend, store_manager, None).await?, + store_factory(&config.backend, store_manager, None, None).await?, )), StoreConfig::compression(config) => Arc::new(CompressionStore::new( *config.clone(), - store_factory(&config.backend, store_manager, None).await?, + store_factory(&config.backend, store_manager, None, None).await?, )?), StoreConfig::dedup(config) => Arc::new(DedupStore::new( config, - store_factory(&config.index_store, store_manager, None).await?, - store_factory(&config.content_store, store_manager, None).await?, + store_factory(&config.index_store, store_manager, None, None).await?, + store_factory(&config.content_store, store_manager, None, None).await?, )), StoreConfig::existence_cache(config) => Arc::new(ExistenceCacheStore::new( config, - store_factory(&config.backend, store_manager, None).await?, + store_factory(&config.backend, store_manager, None, None).await?, )), StoreConfig::completeness_checking(config) => Arc::new(CompletenessCheckingStore::new( - store_factory(&config.backend, store_manager, None).await?, - store_factory(&config.cas_store, store_manager, None).await?, + store_factory(&config.backend, store_manager, None, None).await?, + store_factory(&config.cas_store, store_manager, None, None).await?, )), StoreConfig::fast_slow(config) => Arc::new(FastSlowStore::new( config, - store_factory(&config.fast, store_manager, None).await?, - store_factory(&config.slow, store_manager, None).await?, + store_factory(&config.fast, store_manager, None, None).await?, + store_factory(&config.slow, store_manager, None, None).await?, )), StoreConfig::filesystem(config) => Arc::new(::new(config).await?), StoreConfig::ref_store(config) => Arc::new(RefStore::new(config, Arc::downgrade(store_manager))), StoreConfig::size_partitioning(config) => Arc::new(SizePartitioningStore::new( config, - store_factory(&config.lower_store, store_manager, None).await?, - store_factory(&config.upper_store, store_manager, None).await?, + store_factory(&config.lower_store, store_manager, None, None).await?, + store_factory(&config.upper_store, store_manager, None, None).await?, )), StoreConfig::grpc(config) => Arc::new(GrpcStore::new(config).await?), StoreConfig::noop => Arc::new(NoopStore::new()), @@ -88,7 +90,7 @@ pub fn store_factory<'a>( let stores = config .stores .iter() - .map(|store_config| store_factory(&store_config.store, store_manager, None)) + .map(|store_config| store_factory(&store_config.store, store_manager, None, None)) .collect::>() .try_collect::>() .await?; @@ -98,6 +100,11 @@ pub fn store_factory<'a>( if let Some(store_metrics) = maybe_store_metrics { store.clone().register_metrics(store_metrics); } + + if let Some(health_registry) = maybe_health_registry { + store.clone().register_health(health_registry); + } + Ok(store) }) } diff --git a/nativelink-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs index 5d312e92f8..605acaf2fa 100644 --- a/nativelink-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -14,6 +14,7 @@ use std::ffi::OsString; use std::fmt::{Debug, Formatter}; +use std::io::Write; use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -29,13 +30,16 @@ use nativelink_error::{make_err, make_input_err, Code, Error, ResultExt}; use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::common::{fs, DigestInfo}; use nativelink_util::evicting_map::{EvictingMap, LenEntry}; +use nativelink_util::health_utils::{HealthRegistry, HealthStatus, HealthStatusIndicator}; use nativelink_util::metrics_utils::{Collector, CollectorState, MetricsComponent, Registry}; use nativelink_util::store_trait::{Store, UploadSizeInfo}; +use rand::RngCore; +use tempfile::NamedTempFile; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}; use tokio::task::spawn_blocking; use tokio::time::{sleep, timeout, Sleep}; use tokio_stream::wrappers::ReadDirStream; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use crate::cas_utils::is_zero_digest; @@ -748,6 +752,69 @@ impl Store for FilesystemStore { fn register_metrics(self: Arc, registry: &mut Registry) { registry.register_collector(Box::new(Collector::new(&self))); } + + fn register_health(self: Arc, registry: &mut HealthRegistry) { + registry.register_indicator(self); + } +} + +#[async_trait] +impl HealthStatusIndicator for FilesystemStore { + async fn check_health(self: Arc) -> Result { + let temp_path = &self.shared_context.temp_path; + let temp_file_with_prefix = NamedTempFile::with_prefix_in(".fs_hc_", temp_path); + + let failed_fn = |failure_action, error| { + let message = format!( + "Failed to {:?} temp file in filesystem store health check: {:?}", + failure_action, error + ); + error!(message); + self.make_failed(message.into()) + }; + + match temp_file_with_prefix { + Ok(mut file) => { + let mut data = [0u8; 1_000_000]; + + rand::thread_rng().fill_bytes(&mut data); + + match file.write_all(&data) { + Ok(()) => debug!( + "Successfully wrote to temp file in filesystem store health check {:?}", + file + ), + Err(err) => return Ok(failed_fn("write", err)), + } + + match file.flush() { + Ok(_) => debug!("Successfully flushed temp file in filesystem store health check"), + Err(err) => return Ok(failed_fn("flush", err)), + } + + let file_path: std::path::PathBuf = match file.keep() { + Ok((_, p)) => { + debug!("Successfully kept temp file in filesystem store health check {:?}", p); + p.to_path_buf() + } + Err(err) => return Ok(failed_fn("keep", err.into())), + }; + + debug!("Removing temp file in filesystem store health check {:?}", file_path); + match fs::remove_file(file_path.as_path()).await { + Ok(_) => debug!( + "Successfully removed temp file in filesystem store health check {:?}", + file_path.as_path() + ), + Err(err) => return Ok(failed_fn("remove", err.to_std_err())), + } + + Ok(self.make_ok("Successfully filesystem store health check".into())) + } + + Err(err) => return Ok(failed_fn("create", err)), + } + } } impl MetricsComponent for FilesystemStore { diff --git a/nativelink-util/BUILD.bazel b/nativelink-util/BUILD.bazel index 9d37be3ab2..60e41f75e1 100644 --- a/nativelink-util/BUILD.bazel +++ b/nativelink-util/BUILD.bazel @@ -16,6 +16,7 @@ rust_library( "src/evicting_map.rs", "src/fastcdc.rs", "src/fs.rs", + "src/health_utils.rs", "src/lib.rs", "src/metrics_utils.rs", "src/platform_properties.rs", @@ -27,6 +28,7 @@ rust_library( "src/write_request_stream_wrapper.rs", ], proc_macro_deps = [ + "@crate_index//:async-recursion", "@crate_index//:async-trait", ], visibility = ["//visibility:public"], diff --git a/nativelink-util/Cargo.toml b/nativelink-util/Cargo.toml index 3bcf20bd76..d6dc93f115 100644 --- a/nativelink-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -9,6 +9,7 @@ nativelink-error = { path = "../nativelink-error" } nativelink-proto = { path = "../nativelink-proto" } async-lock = "3.2.0" +async-recursion = "1.0.5" async-trait = "0.1.74" blake3 = "1.5.0" bytes = "1.5.0" diff --git a/nativelink-util/src/health_utils.rs b/nativelink-util/src/health_utils.rs new file mode 100644 index 0000000000..0e86329cb1 --- /dev/null +++ b/nativelink-util/src/health_utils.rs @@ -0,0 +1,144 @@ +use std::borrow::Cow; +use std::fmt::Debug; +use std::marker::Send; +use std::sync::Arc; + +use async_recursion::async_recursion; +use async_trait::async_trait; +use nativelink_error::Error; + +type HealthComponent = Cow<'static, str>; +type TypeName = Cow<'static, str>; +pub type Message = Cow<'static, str>; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum HealthStatus { + Ok(TypeName, Message), + Initializing(TypeName, Message), + Warning(TypeName, Message), + Failed(TypeName, Message), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct HealthStatusDescription { + pub component: HealthComponent, + pub status: HealthStatus, +} + +#[async_trait] +pub trait HealthStatusIndicator: Sync + Send + Unpin { + fn type_name(&self) -> TypeName { + Cow::Borrowed(std::any::type_name::()) + } + + async fn check_health(self: Arc) -> Result { + Ok(self.make_ok("ok".into())) + } + + fn make_ok(&self, message: Message) -> HealthStatus { + HealthStatus::Ok(self.type_name(), message) + } + + fn make_initializing(&self, message: Message) -> HealthStatus { + HealthStatus::Initializing(self.type_name(), message) + } + + fn make_warning(&self, message: Message) -> HealthStatus { + HealthStatus::Warning(self.type_name(), message) + } + + fn make_failed(&self, message: Message) -> HealthStatus { + HealthStatus::Failed(self.type_name(), message) + } +} + +#[derive(Default, Clone)] +pub struct HealthRegistry { + component: HealthComponent, + indicators: Vec>, + registries: Vec, +} + +impl HealthRegistry { + pub fn new(component: HealthComponent) -> Self { + Self { + component, + ..Default::default() + } + } + + pub fn register_indicator(&mut self, indicator: Arc) { + self.indicators.push(indicator); + } + + pub fn add_dependency(&mut self, component: HealthComponent) -> &mut Self { + let dependency = HealthRegistry::new(component); + + self.registries.push(dependency); + self.registries + .last_mut() + .expect("dependencies should not to be empty.") + } + + #[async_recursion] + async fn flatten( + &mut self, + results: &mut Vec, + parent_component: &HealthComponent, + component: &HealthComponent, + indicators: &Vec>, + registries: &Vec, + ) -> Result<(), Error> { + let component_name: Cow<'static, str> = Cow::Owned(format!("{parent_component}/{component}")); + for indicator in indicators { + let result = indicator.clone().check_health().await; + + let health_status = match result { + Ok(health_status) => HealthStatusDescription { + component: component_name.clone(), + status: health_status, + }, + Err(error) => HealthStatusDescription { + component: component_name.clone(), + status: indicator.make_failed(format!("health check failed: {error}").into()), + }, + }; + + results.push(health_status); + } + + for registry in registries { + let _ = self + .clone() + .flatten( + results, + &component_name, + ®istry.component, + ®istry.indicators, + ®istry.registries, + ) + .await; + } + + Ok(()) + } + + pub async fn flatten_indicators(&mut self) -> Vec { + let mut health_status_results = Vec::new(); + let parent_component: HealthComponent = "".into(); + let component = &self.component; + let indicators = &self.indicators; + let registries = &self.registries; + let _ = self + .clone() + .flatten( + &mut health_status_results, + &parent_component, + component, + indicators, + registries, + ) + .await; + health_status_results + } +} diff --git a/nativelink-util/src/lib.rs b/nativelink-util/src/lib.rs index 3dad898e82..04bf0ebf58 100644 --- a/nativelink-util/src/lib.rs +++ b/nativelink-util/src/lib.rs @@ -19,6 +19,7 @@ pub mod digest_hasher; pub mod evicting_map; pub mod fastcdc; pub mod fs; +pub mod health_utils; pub mod metrics_utils; pub mod platform_properties; pub mod resource_info; diff --git a/nativelink-util/src/store_trait.rs b/nativelink-util/src/store_trait.rs index b918391068..87ff456b8f 100644 --- a/nativelink-util/src/store_trait.rs +++ b/nativelink-util/src/store_trait.rs @@ -24,6 +24,7 @@ use serde::{Deserialize, Serialize}; use crate::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; use crate::common::DigestInfo; +use crate::health_utils::HealthRegistry; use crate::metrics_utils::Registry; #[derive(Debug, PartialEq, Copy, Clone, Serialize, Deserialize)] @@ -180,4 +181,6 @@ pub trait Store: Sync + Send + Unpin { /// Register any metrics that this store wants to expose to the Prometheus. fn register_metrics(self: Arc, _registry: &mut Registry) {} + + fn register_health(self: Arc, _registry: &mut HealthRegistry) {} } diff --git a/nativelink-util/tests/health_utils_test.rs b/nativelink-util/tests/health_utils_test.rs new file mode 100644 index 0000000000..885d9b3592 --- /dev/null +++ b/nativelink-util/tests/health_utils_test.rs @@ -0,0 +1,179 @@ +use std::collections::HashSet; +use std::iter::FromIterator; +use std::sync::{Arc, Mutex}; + +use nativelink_error::Error; + +#[cfg(test)] +mod health_utils_tests { + + use nativelink_util::health_utils::*; + use pretty_assertions::assert_eq; + + use super::*; + + #[tokio::test] + async fn create_empty_indicator() -> Result<(), Error> { + let health_registery = Arc::new(Mutex::new(HealthRegistry::new("nativelink".into()))); + + let health_status = health_registery.lock().unwrap().flatten_indicators().await; + assert_eq!(health_status.len(), 0); + + Ok(()) + } + + #[tokio::test] + async fn create_register_indicator() -> Result<(), Error> { + generate_health_status_indicator!(MockComponentImpl, HealthStatus::Ok, "ok"); + + let health_registery = Arc::new(Mutex::new(HealthRegistry::new("nativelink".into()))); + let mut health_registery = health_registery.lock().unwrap(); + + health_registery.register_indicator(Arc::new(MockComponentImpl {})); + + let health_status = health_registery.flatten_indicators().await; + assert_eq!(health_status.len(), 1); + assert_eq!( + health_status, + vec![HealthStatusDescription { + component: "/nativelink".into(), + status: HealthStatus::Ok("MockComponentImpl".into(), "ok".into()), + }] + ); + + Ok(()) + } + + #[tokio::test] + async fn create_add_dependency() -> Result<(), Error> { + generate_health_status_indicator!(MockComponentImpl, HealthStatus::Ok, "ok"); + + let health_registery = Arc::new(Mutex::new(HealthRegistry::new("nativelink".into()))); + let mut health_registery = health_registery.lock().unwrap(); + + health_registery.register_indicator(Arc::new(MockComponentImpl {})); + + let dependency1_registry = health_registery.add_dependency("dependency1".into()); + + dependency1_registry.register_indicator(Arc::new(MockComponentImpl {})); + + let health_status = health_registery.flatten_indicators().await; + // println!("health_status: {:?}", health_status); + assert_eq!(health_status.len(), 2); + let expected_health_status = vec![ + HealthStatusDescription { + component: "/nativelink".into(), + status: HealthStatus::Ok("MockComponentImpl".into(), "ok".into()), + }, + HealthStatusDescription { + component: "/nativelink/dependency1".into(), + status: HealthStatus::Ok("MockComponentImpl".into(), "ok".into()), + }, + ]; + + assert_eq!(health_status, expected_health_status); + + Ok(()) + } + + #[tokio::test] + async fn create_multiple_indicators_same_level() -> Result<(), Error> { + generate_health_status_indicator!(MockComponentImpl1, HealthStatus::Ok, "ok"); + generate_health_status_indicator!(MockComponentImpl2, HealthStatus::Ok, "ok"); + generate_health_status_indicator!(MockComponentImpl3, HealthStatus::Ok, "ok"); + + let health_registery = Arc::new(Mutex::new(HealthRegistry::new("nativelink".into()))); + let mut health_registery = health_registery.lock().unwrap(); + + health_registery.register_indicator(Arc::new(MockComponentImpl1 {})); + health_registery.register_indicator(Arc::new(MockComponentImpl2 {})); + health_registery.register_indicator(Arc::new(MockComponentImpl3 {})); + + let health_status = health_registery.flatten_indicators().await; + + assert_eq!(health_status.len(), 3); + let expected_health_status = vec_to_set(vec![ + HealthStatusDescription { + component: "/nativelink".into(), + status: HealthStatus::Ok("MockComponentImpl1".into(), "ok".into()), + }, + HealthStatusDescription { + component: "/nativelink".into(), + status: HealthStatus::Ok("MockComponentImpl2".into(), "ok".into()), + }, + HealthStatusDescription { + component: "/nativelink".into(), + status: HealthStatus::Ok("MockComponentImpl3".into(), "ok".into()), + }, + ]); + + assert_eq!(vec_to_set(health_status), expected_health_status); + + Ok(()) + } + + #[tokio::test] + async fn create_multiple_indicators_nested_levels() -> Result<(), Error> { + generate_health_status_indicator!(MockComponentImpl1, HealthStatus::Ok, "ok"); + generate_health_status_indicator!(MockComponentImpl2, HealthStatus::Ok, "ok"); + generate_health_status_indicator!(MockComponentImpl3, HealthStatus::Ok, "ok"); + + let health_registery = Arc::new(Mutex::new(HealthRegistry::new("nativelink".into()))); + let mut health_registery = health_registery.lock().unwrap(); + + health_registery + .add_dependency("dependency1".into()) + .register_indicator(Arc::new(MockComponentImpl1 {})); + + health_registery + .add_dependency("dependency2".into()) + .register_indicator(Arc::new(MockComponentImpl2 {})); + + health_registery + .add_dependency("dependency3".into()) + .register_indicator(Arc::new(MockComponentImpl3 {})); + + let health_status = health_registery.flatten_indicators().await; + + assert_eq!(health_status.len(), 3); + let expected_health_status = vec_to_set(vec![ + HealthStatusDescription { + component: "/nativelink/dependency1".into(), + status: HealthStatus::Ok("MockComponentImpl1".into(), "ok".into()), + }, + HealthStatusDescription { + component: "/nativelink/dependency2".into(), + status: HealthStatus::Ok("MockComponentImpl2".into(), "ok".into()), + }, + HealthStatusDescription { + component: "/nativelink/dependency3".into(), + status: HealthStatus::Ok("MockComponentImpl3".into(), "ok".into()), + }, + ]); + + assert_eq!(vec_to_set(health_status), expected_health_status); + + Ok(()) + } + + #[macro_export] + macro_rules! generate_health_status_indicator { + ($struct_name:ident, $health_status:expr, $status_msg:expr) => { + struct $struct_name; + + #[async_trait::async_trait] + impl HealthStatusIndicator for $struct_name { + async fn check_health(self: Arc) -> Result { + Ok($health_status( + stringify!($struct_name).into(), + $status_msg.into(), + )) + } + } + }; + } + + fn vec_to_set(vec: Vec) -> HashSet { + HashSet::from_iter(vec) + } +} diff --git a/src/bin/nativelink.rs b/src/bin/nativelink.rs index 9c01d330b6..9f75fcf733 100644 --- a/src/bin/nativelink.rs +++ b/src/bin/nativelink.rs @@ -41,6 +41,7 @@ use nativelink_store::default_store_factory::store_factory; use nativelink_store::store_manager::StoreManager; use nativelink_util::common::fs::{set_idle_file_descriptor_timeout, set_open_file_limit}; use nativelink_util::digest_hasher::{set_default_digest_hasher_func, DigestHasherFunc}; +use nativelink_util::health_utils::HealthRegistry; use nativelink_util::metrics_utils::{ set_metrics_enabled_for_this_thread, Collector, CollectorState, Counter, MetricsComponent, Registry, }; @@ -87,17 +88,27 @@ struct Args { async fn inner_main(cfg: CasConfig, server_start_timestamp: u64) -> Result<(), Box> { let mut root_metrics_registry = ::with_prefix("nativelink"); + let health_registery = Arc::new(AsyncMutex::new(HealthRegistry::new("nativelink".into()))); let store_manager = Arc::new(StoreManager::new()); { + let mut health_registery_lock = health_registery.lock().await; let root_store_metrics = root_metrics_registry.sub_registry_with_prefix("stores"); + for (name, store_cfg) in cfg.stores { + let health_component_name = format!("stores/{name}"); + let health_register_store = health_registery_lock.add_dependency(health_component_name.into()); let store_metrics = root_store_metrics.sub_registry_with_prefix(&name); store_manager.add_store( &name, - store_factory(&store_cfg, &store_manager, Some(store_metrics)) - .await - .err_tip(|| format!("Failed to create store '{name}'"))?, + store_factory( + &store_cfg, + &store_manager, + Some(store_metrics), + Some(health_register_store), + ) + .await + .err_tip(|| format!("Failed to create store '{name}'"))?, ); } } @@ -349,12 +360,42 @@ async fn inner_main(cfg: CasConfig, server_start_timestamp: u64) -> Result<(), B ); let root_metrics_registry = root_metrics_registry.clone(); - + let health_registery_status = health_registery.clone(); let mut svc = Router::new() // This is the default service that executes if no other endpoint matches. .fallback_service(tonic_services.into_service().map_err(|e| panic!("{e}"))) - // This is a generic endpoint used to check if the server is up. - .route_service("/status", axum::routing::get(move || async move { "Ok".to_string() })); + .route_service( + "/status", + axum::routing::get(move || async move { + fn error_to_response(e: E) -> Response { + let mut response = Response::new(format!("Error: {e:?}")); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + response + } + + spawn_blocking(move || { + let mut buf = String::new(); + + let indicators = futures::executor::block_on(async { + let mut health_registery_status_guard = health_registery_status.lock().await; + health_registery_status_guard.flatten_indicators().await + }); + + for indicator in indicators { + buf.push_str(&format!("{:?}\n", indicator)); + } + + let mut response = Response::new(buf); + response.headers_mut().insert( + hyper::header::CONTENT_TYPE, + hyper::header::HeaderValue::from_static("text/plain; version=0.0.4; charset=utf-8"), + ); + response + }) + .await + .unwrap_or_else(error_to_response) + }), + ); if let Some(prometheus_cfg) = services.experimental_prometheus { fn error_to_response(e: E) -> Response {