Add Redis Store (TraceMachina#393)

Signed-off-by: Blake Hatch <[email protected]>

Cargo.toml

@@ -37,6 +37,9 @@ nativelink-worker = { path = "nativelink-worker" }
 
 async-lock = "3.3.0"
 axum = "0.6.20"
+
+# The cc crate must be pinned to be compatible with the zig-cc toolchain.
+cc = "1.0.83"
 clap = { version = "4.5.4", features = ["derive"] }
 console-subscriber = { version = "0.2.0" }
 futures = "0.3.30"

nativelink-config/src/stores.rs

@@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::serde_utils::{
     convert_numeric_with_shellexpand, convert_optional_string_with_shellexpand,
-    convert_string_with_shellexpand,
+    convert_string_with_shellexpand, convert_vec_string_with_shellexpand,
 };
 
 /// Name of the store. This type will be used when referencing a store
@@ -163,6 +163,13 @@ pub enum StoreConfig {
     /// side effects and is the most efficient way to use it.
     grpc(GrpcStore),
 
+    /// Stores data in any stores compatible with Redis APIs.
+    ///
+    /// Pairs well with SizePartitioning and/or FastSlow stores.
+    /// Ideal for accepting small object sizes as most redis store
+    /// services have a max file upload of between 256Mb-512Mb.
+    redis_store(RedisStore),
+
     /// Noop store is a store that sends streams into the void and all data
     /// retrieval will return 404 (NotFound). This can be useful for cases
     /// where you may need to partition your data and part of your data needs
@@ -597,6 +604,28 @@ pub enum ErrorCode {
     // Note: This list is duplicated from nativelink-error/lib.rs.
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct RedisStore {
+    /// The hostname or IP address of the Redis server.
+    /// Ex: ["redis://username:password@redis-server-url:6380/99"]
+    /// 99 Represents database ID, 6380 represents the port.
+    // Note: This is currently one address but supports multile for clusters.
+    #[serde(deserialize_with = "convert_vec_string_with_shellexpand")]
+    pub addresses: Vec<String>,
+
+    /// The response timeout for the Redis connection in seconds.
+    ///
+    /// Default: 10
+    #[serde(default)]
+    pub response_timeout_s: u64,
+
+    /// The connection timeout for the Redis connection in seconds.
+    ///
+    /// Default: 10
+    #[serde(default)]
+    pub connection_timeout_s: u64,
+}
+
 /// Retry configuration. This configuration is exponential and each iteration
 /// a jitter as a percentage is applied of the calculated delay. For example:
 /// ```rust,ignore

nativelink-store/BUILD.bazel

@@ -22,6 +22,7 @@ rust_library(
         "src/lib.rs",
         "src/memory_store.rs",
         "src/noop_store.rs",
+        "src/redis_store.rs",
         "src/ref_store.rs",
         "src/s3_store.rs",
         "src/shard_store.rs",
@@ -38,6 +39,7 @@ rust_library(
         "//nativelink-error",
         "//nativelink-proto",
         "//nativelink-util",
+        "@crates//:arc-cell",
         "@crates//:async-lock",
         "@crates//:aws-config",
         "@crates//:aws-sdk-s3",
@@ -56,6 +58,7 @@ rust_library(
         "@crates//:parking_lot",
         "@crates//:prost",
         "@crates//:rand",
+        "@crates//:redis",
         "@crates//:serde",
         "@crates//:sha2",
         "@crates//:shellexpand",
@@ -81,6 +84,7 @@ rust_test_suite(
         "tests/fast_slow_store_test.rs",
         "tests/filesystem_store_test.rs",
         "tests/memory_store_test.rs",
+        "tests/redis_store_test.rs",
         "tests/ref_store_test.rs",
         "tests/s3_store_test.rs",
         "tests/shard_store_test.rs",
@@ -105,16 +109,20 @@ rust_test_suite(
         "@crates//:bytes",
         "@crates//:filetime",
         "@crates//:futures",
+        "@crates//:hex",
         "@crates//:http",
         "@crates//:http-body",
         "@crates//:hyper",
         "@crates//:memory-stats",
         "@crates//:once_cell",
         "@crates//:pretty_assertions",
         "@crates//:rand",
+        "@crates//:redis",
+        "@crates//:redis-test",
         "@crates//:sha2",
         "@crates//:tokio",
         "@crates//:tokio-stream",
+        "@crates//:uuid",
     ],
 )
 

nativelink-store/Cargo.toml

@@ -9,6 +9,7 @@ nativelink-config = { path = "../nativelink-config" }
 nativelink-util = { path = "../nativelink-util" }
 nativelink-proto = { path = "../nativelink-proto" }
 
+arc-cell = "0.3.3"
 async-lock = "3.3.0"
 async-trait = "0.1.80"
 aws-config = "1.1.9"
@@ -28,6 +29,7 @@ lz4_flex = "0.11.3"
 parking_lot = "0.12.1"
 prost = "0.12.4"
 rand = "0.8.5"
+redis = { version = "0.25.2", features = ["tokio-comp", "tokio-rustls-comp", "connection-manager"] }
 serde = "1.0.197"
 sha2 = "0.10.8"
 shellexpand = "3.1.0"
@@ -40,6 +42,7 @@ tracing = "0.1.40"
 uuid = { version = "1.8.0", features = ["v4"] }
 
 [dev-dependencies]
+redis-test = { version = "0.4.0", features = ["aio"] }
 pretty_assertions = "1.4.0"
 memory-stats = "1.1.0"
 once_cell = "1.19.0"

nativelink-store/src/cas_utils.rs

@@ -14,7 +14,7 @@
 
 use nativelink_util::common::DigestInfo;
 
-const ZERO_BYTE_DIGESTS: [DigestInfo; 2] = [
+pub const ZERO_BYTE_DIGESTS: [DigestInfo; 2] = [
     // Sha256 hash of zero bytes.
     DigestInfo::new(
         [

nativelink-store/src/default_store_factory.rs

@@ -32,6 +32,7 @@ use crate::filesystem_store::FilesystemStore;
 use crate::grpc_store::GrpcStore;
 use crate::memory_store::MemoryStore;
 use crate::noop_store::NoopStore;
+use crate::redis_store::RedisStore;
 use crate::ref_store::RefStore;
 use crate::s3_store::S3Store;
 use crate::shard_store::ShardStore;
@@ -51,6 +52,7 @@ pub fn store_factory<'a>(
         let store: Arc<dyn Store> = match backend {
             StoreConfig::memory(config) => Arc::new(MemoryStore::new(config)),
             StoreConfig::experimental_s3_store(config) => Arc::new(S3Store::new(config).await?),
+            StoreConfig::redis_store(config) => Arc::new(RedisStore::new(config)?),
             StoreConfig::verify(config) => Arc::new(VerifyStore::new(
                 config,
                 store_factory(&config.backend, store_manager, None, None).await?,

nativelink-store/src/lib.rs

@@ -24,6 +24,7 @@ pub mod filesystem_store;
 pub mod grpc_store;
 pub mod memory_store;
 pub mod noop_store;
+pub mod redis_store;
 pub mod ref_store;
 pub mod s3_store;
 pub mod shard_store;