From 2ae708152ed32579b484344eb6531dd1c991ac0b Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 08:04:19 -0400 Subject: [PATCH 01/16] Add scaffolding. --- Cargo.lock | 150 ++++++++++++++++++ Cargo.toml | 1 + components/clp-rust-utils/src/lib.rs | 1 + components/clp-rust-utils/src/task_io.rs | 1 + .../clp-rust-utils/src/task_io/compression.rs | 41 +++++ components/compression-coordinator/Cargo.toml | 15 ++ .../src/compression_job_submitter.rs | 76 +++++++++ .../compression-coordinator/src/error.rs | 5 + components/compression-coordinator/src/lib.rs | 8 + 9 files changed, 298 insertions(+) create mode 100644 components/clp-rust-utils/src/task_io.rs create mode 100644 components/clp-rust-utils/src/task_io/compression.rs create mode 100644 components/compression-coordinator/Cargo.toml create mode 100644 components/compression-coordinator/src/compression_job_submitter.rs create mode 100644 components/compression-coordinator/src/error.rs create mode 100644 components/compression-coordinator/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 46840eb622..1eba6ec59d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -988,6 +988,17 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "compression-coordinator" +version = "0.13.1-dev" +dependencies = [ + "async-trait", + "clp-rust-utils", + "serde", + "spider-core", + "thiserror", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2042,6 +2053,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.9.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -2872,6 +2896,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3692,6 +3736,52 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spider-core" +version = "0.1.0" +source = "git+https://github.com/y-scope/spider.git?branch=main#983dde9bc946c1683ffd202055924c68ae3a3492" +dependencies = [ + "non-empty-string", + "rand 0.9.4", + "rmp-serde", + "semver", + "serde", + "serde_json", + "spider-derive", + "spider-utils", + "sqlx", + "strum", + "thiserror", + "zstd", +] + +[[package]] +name = "spider-derive" +version = "0.1.0" +source = "git+https://github.com/y-scope/spider.git?branch=main#983dde9bc946c1683ffd202055924c68ae3a3492" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "spider-utils" +version = "0.1.0" +source = "git+https://github.com/y-scope/spider.git?branch=main#983dde9bc946c1683ffd202055924c68ae3a3492" +dependencies = [ + "non-empty-string", + "rand 0.9.4", + "rmp-serde", + "serde", + "thiserror", + "tokio", + "tonic", + "tracing-appender", + "tracing-subscriber", + "yaml_serde", +] + [[package]] name = "spin" version = "0.9.8" @@ -4232,6 +4322,35 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2 0.4.14", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2 0.6.3", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -4240,9 +4359,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -5172,3 +5294,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index a3a8bef443..c8f8676876 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "components/api-server", "components/clp-rust-utils", + "components/compression-coordinator", "components/log-ingestor" ] resolver = "3" diff --git a/components/clp-rust-utils/src/lib.rs b/components/clp-rust-utils/src/lib.rs index b3e0d6af7d..e6091b96f4 100644 --- a/components/clp-rust-utils/src/lib.rs +++ b/components/clp-rust-utils/src/lib.rs @@ -8,6 +8,7 @@ pub mod logging; pub mod s3; pub mod serde; pub mod sqs; +pub mod task_io; pub mod telemetry; pub mod types; diff --git a/components/clp-rust-utils/src/task_io.rs b/components/clp-rust-utils/src/task_io.rs new file mode 100644 index 0000000000..376ef623d5 --- /dev/null +++ b/components/clp-rust-utils/src/task_io.rs @@ -0,0 +1 @@ +pub mod compression; diff --git a/components/clp-rust-utils/src/task_io/compression.rs b/components/clp-rust-utils/src/task_io/compression.rs new file mode 100644 index 0000000000..db6c5ca627 --- /dev/null +++ b/components/clp-rust-utils/src/task_io/compression.rs @@ -0,0 +1,41 @@ +//! Protocol types exchanged with the Spider (Huntsman) tasks that run CLP S3 compression jobs. + +use non_empty_string::NonEmptyString; +use serde::{Deserialize, Serialize}; + +use crate::clp_config::AwsAuthentication; + +/// `clp-s` tuning and engine options for a compression job. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClpSCompressionOption { + pub target_encoded_size: u64, + pub compression_level: i32, + pub timestamp_key: Option, +} + +/// Input source for a compression task. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct S3InputSource { + pub endpoint_url: Option, + pub region_code: Option, + pub bucket: NonEmptyString, + pub aws_authentication: AwsAuthentication, + pub object_keys: Vec, +} + +/// Output of a compression task. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CompressionTaskOutput { + pub dataset: Option, + pub archives: Vec, +} + +/// Metadata of an archive produced by `clp-s`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArchiveMetadata { + pub id: String, + pub begin_timestamp: i64, + pub end_timestamp: i64, + pub size: i64, + pub uncompressed_size: i64, +} diff --git a/components/compression-coordinator/Cargo.toml b/components/compression-coordinator/Cargo.toml new file mode 100644 index 0000000000..861b596414 --- /dev/null +++ b/components/compression-coordinator/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "compression-coordinator" +version = "0.13.1-dev" +edition = "2024" + +[lib] +name = "compression_coordinator" +path = "src/lib.rs" + +[dependencies] +async-trait = "0.1.89" +clp-rust-utils = { path = "../clp-rust-utils" } +serde = { version = "1.0.228", features = ["derive"] } +spider-core = { git = "https://github.com/y-scope/spider.git", branch = "main" } +thiserror = "2.0.18" diff --git a/components/compression-coordinator/src/compression_job_submitter.rs b/components/compression-coordinator/src/compression_job_submitter.rs new file mode 100644 index 0000000000..d33aadfc8c --- /dev/null +++ b/components/compression-coordinator/src/compression_job_submitter.rs @@ -0,0 +1,76 @@ +//! The compression-job-submission API trait for driving CLP S3 compression jobs on a Spider +//! (Huntsman) cluster. + +use async_trait::async_trait; +use clp_rust_utils::task_io::compression::{ClpSCompressionOption, S3InputSource}; +use serde::{Deserialize, Serialize}; +use spider_core::types::id::{JobId, ResourceGroupId}; +use clp_rust_utils::job_config::CompressionJobId; +use crate::error::Error; + +/// The terminal outcome of a compression job. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum CompressionJobOutcome { + /// The job completed successfully. + Succeeded, + + /// The job failed with the given error. + Failed { error_message: String }, + + /// The job was cancelled before reaching completion. + Cancelled, +} + +/// Drives CLP S3 compression jobs on a Spider (Huntsman) cluster. +#[async_trait] +pub trait S3CompressionJobSubmitter: Clone + Send + Sync { + /// Builds the compression task graph for `input_sources` and registers it with Spider, without + /// starting it. + /// + /// # Parameters + /// + /// * `compression_job_id` - The unique ID of the CLP compression job. + /// * `resource_group_id` - The Spider resource group to register the job under. + /// * `clp_s_option` - `clp-s` tuning options shared by every task in the job. + /// * `dataset` - The dataset to compress into. + /// * `input_sources` - S3 input sources to compress. Each represents an input to a compression + /// task. + /// + /// # Returns + /// + /// The job ID issued by Spider on success. + /// + /// # Errors + /// + /// Implementations must document their error conditions. + async fn submit_s3_compression_job( + &self, + compression_job_id: CompressionJobId, + resource_group_id: ResourceGroupId, + clp_s_option: ClpSCompressionOption, + dataset: Option, + input_sources: Vec, + ) -> Result; + + /// Idempotently starts the job identified by `spider_job_id` (only if it hasn't already been + /// started) and waits until it reaches a terminal state. + /// + /// Safe to call regardless of whether the job is not-yet-started, already running, or already + /// terminal. + /// + /// # Parameters + /// + /// * `spider_job_id` - The job to start (if needed) and wait on. + /// + /// # Returns + /// + /// The job's terminal outcome on success. + /// + /// # Errors + /// + /// Implementations must document their error conditions. + async fn run_s3_compression_job_to_completion( + &self, + spider_job_id: JobId, + ) -> Result; +} diff --git a/components/compression-coordinator/src/error.rs b/components/compression-coordinator/src/error.rs new file mode 100644 index 0000000000..027c335c9c --- /dev/null +++ b/components/compression-coordinator/src/error.rs @@ -0,0 +1,5 @@ +//! The crate-level error type for the compression coordinator. + +/// Errors returned by the compression coordinator. +#[derive(Debug, thiserror::Error)] +pub enum Error {} diff --git a/components/compression-coordinator/src/lib.rs b/components/compression-coordinator/src/lib.rs new file mode 100644 index 0000000000..e0b157379d --- /dev/null +++ b/components/compression-coordinator/src/lib.rs @@ -0,0 +1,8 @@ +//! The compression-job-submission API trait for driving CLP S3 compression jobs on a Spider +//! (Huntsman) cluster. + +pub mod compression_job_submitter; +mod error; + +pub use compression_job_submitter::{CompressionJobOutcome, S3CompressionJobSubmitter}; +pub use error::Error; From 8eba802a7dbd61bfe48e4f2669177219dce7128e Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 08:47:57 -0400 Subject: [PATCH 02/16] Move to a dir. --- .../mod.rs} | 9 +++++++-- .../src/compression_job_submitter/spider.rs | 1 + components/compression-coordinator/src/lib.rs | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) rename components/compression-coordinator/src/{compression_job_submitter.rs => compression_job_submitter/mod.rs} (94%) create mode 100644 components/compression-coordinator/src/compression_job_submitter/spider.rs diff --git a/components/compression-coordinator/src/compression_job_submitter.rs b/components/compression-coordinator/src/compression_job_submitter/mod.rs similarity index 94% rename from components/compression-coordinator/src/compression_job_submitter.rs rename to components/compression-coordinator/src/compression_job_submitter/mod.rs index d33aadfc8c..100969f8f1 100644 --- a/components/compression-coordinator/src/compression_job_submitter.rs +++ b/components/compression-coordinator/src/compression_job_submitter/mod.rs @@ -1,11 +1,16 @@ //! The compression-job-submission API trait for driving CLP S3 compression jobs on a Spider //! (Huntsman) cluster. +mod spider; + use async_trait::async_trait; -use clp_rust_utils::task_io::compression::{ClpSCompressionOption, S3InputSource}; +use clp_rust_utils::{ + job_config::CompressionJobId, + task_io::compression::{ClpSCompressionOption, S3InputSource}, +}; use serde::{Deserialize, Serialize}; use spider_core::types::id::{JobId, ResourceGroupId}; -use clp_rust_utils::job_config::CompressionJobId; + use crate::error::Error; /// The terminal outcome of a compression job. diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -0,0 +1 @@ + diff --git a/components/compression-coordinator/src/lib.rs b/components/compression-coordinator/src/lib.rs index e0b157379d..47921c0041 100644 --- a/components/compression-coordinator/src/lib.rs +++ b/components/compression-coordinator/src/lib.rs @@ -4,5 +4,4 @@ pub mod compression_job_submitter; mod error; -pub use compression_job_submitter::{CompressionJobOutcome, S3CompressionJobSubmitter}; pub use error::Error; From f919f14f27a07f3660f8d246ae7435d2ff3287b1 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 09:46:20 -0400 Subject: [PATCH 03/16] Make the poll backoffs parameters of the job-completion API. The initial and maximum job-state poll backoffs were fixed constants inside the Spider implementation, so a caller had no way to tune how aggressively a job is polled. Both are now `Duration` parameters of `run_s3_compression_job_to_completion`, leaving the pacing decision with the caller that knows how long its jobs typically run. --- .../src/compression_job_submitter/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/compression-coordinator/src/compression_job_submitter/mod.rs b/components/compression-coordinator/src/compression_job_submitter/mod.rs index 100969f8f1..07fb09a871 100644 --- a/components/compression-coordinator/src/compression_job_submitter/mod.rs +++ b/components/compression-coordinator/src/compression_job_submitter/mod.rs @@ -3,6 +3,8 @@ mod spider; +use std::time::Duration; + use async_trait::async_trait; use clp_rust_utils::{ job_config::CompressionJobId, @@ -66,6 +68,8 @@ pub trait S3CompressionJobSubmitter: Clone + Send + Sync { /// # Parameters /// /// * `spider_job_id` - The job to start (if needed) and wait on. + /// * `initial_poll_backoff` - The delay before the first job-state poll. + /// * `max_poll_backoff` - The cap on the delay between job-state polls. /// /// # Returns /// @@ -77,5 +81,7 @@ pub trait S3CompressionJobSubmitter: Clone + Send + Sync { async fn run_s3_compression_job_to_completion( &self, spider_job_id: JobId, + initial_poll_backoff: Duration, + max_poll_backoff: Duration, ) -> Result; } From 21a51c9e000fb5c1d5f69054fa3a0e36a733518e Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 09:50:04 -0400 Subject: [PATCH 04/16] Done impl --- Cargo.lock | 194 +++++++++++++++++- components/compression-coordinator/Cargo.toml | 4 + .../src/compression_job_submitter/spider.rs | 161 +++++++++++++++ .../compression-coordinator/src/error.rs | 13 +- 4 files changed, 365 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1eba6ec59d..6ab14c17e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,9 +994,13 @@ version = "0.13.1-dev" dependencies = [ "async-trait", "clp-rust-utils", + "rmp-serde", "serde", + "spider-client", "spider-core", "thiserror", + "tokio", + "tracing", ] [[package]] @@ -1474,7 +1478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1521,6 +1525,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flume" version = "0.11.1" @@ -2083,7 +2093,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2372,6 +2382,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -2633,6 +2649,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "non-empty-string" version = "0.2.6" @@ -2896,6 +2918,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -3031,19 +3064,40 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.117", + "tempfile", +] + [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -3052,6 +3106,35 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quote" version = "1.0.45" @@ -3311,6 +3394,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.21.12" @@ -3736,6 +3832,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spider-client" +version = "0.1.0" +source = "git+https://github.com/y-scope/spider.git?branch=main#983dde9bc946c1683ffd202055924c68ae3a3492" +dependencies = [ + "spider-core", + "spider-proto-rust", + "spider-utils", + "thiserror", + "tokio", + "tonic", +] + [[package]] name = "spider-core" version = "0.1.0" @@ -3765,6 +3874,21 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "spider-proto-rust" +version = "0.1.0" +source = "git+https://github.com/y-scope/spider.git?branch=main#983dde9bc946c1683ffd202055924c68ae3a3492" +dependencies = [ + "prost", + "spider-core", + "spider-utils", + "thiserror", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tracing", +] + [[package]] name = "spider-utils" version = "0.1.0" @@ -4125,6 +4249,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -4351,6 +4488,45 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.117", + "tempfile", + "tonic-build", +] + [[package]] name = "tower" version = "0.5.3" @@ -4520,6 +4696,12 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" diff --git a/components/compression-coordinator/Cargo.toml b/components/compression-coordinator/Cargo.toml index 861b596414..5bb902221d 100644 --- a/components/compression-coordinator/Cargo.toml +++ b/components/compression-coordinator/Cargo.toml @@ -10,6 +10,10 @@ path = "src/lib.rs" [dependencies] async-trait = "0.1.89" clp-rust-utils = { path = "../clp-rust-utils" } +rmp-serde = "1.3.1" serde = { version = "1.0.228", features = ["derive"] } +spider-client = { git = "https://github.com/y-scope/spider.git", branch = "main" } spider-core = { git = "https://github.com/y-scope/spider.git", branch = "main" } thiserror = "2.0.18" +tokio = { version = "1.52.3", features = ["time"] } +tracing = "0.1.44" diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index 8b13789179..88d0aa5717 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -1 +1,162 @@ +//! [`S3CompressionJobSubmitter`] implementation for [`spider_client::SpiderClient`]. +use std::time::Duration; + +use async_trait::async_trait; +use clp_rust_utils::{ + job_config::CompressionJobId, + task_io::compression::{ClpSCompressionOption, S3InputSource}, +}; +use spider_client::{SpiderClient, error::ClientError}; +use spider_core::{ + job::JobState, + task::{ + DataTypeDescriptor, + ExecutionPolicy, + TaskDescriptor, + TaskGraph, + TdlContext, + TerminationTaskDescriptor, + TimeoutPolicy, + ValueTypeDescriptor, + }, + types::{ + id::{JobId, ResourceGroupId}, + io::TaskInput, + }, +}; + +use crate::{ + compression_job_submitter::{CompressionJobOutcome, S3CompressionJobSubmitter}, + error::Error, +}; + +#[async_trait] +impl S3CompressionJobSubmitter for SpiderClient { + /// # Errors + /// + /// Returns an error if: + /// + /// * Forwards [`TaskGraph::new`]'s return values on failure. + /// * Forwards [`ValueTypeDescriptor::struct_from_name`]'s return values on failure. + /// * Forwards [`TaskGraph::insert_task`]'s return values on failure. + /// * Forwards [`rmp_serde::to_vec`]'s return values on failure. + /// * Forwards [`SpiderClient::submit_job`]'s return values on failure. + async fn submit_s3_compression_job( + &self, + compression_job_id: CompressionJobId, + resource_group_id: ResourceGroupId, + clp_s_option: ClpSCompressionOption, + dataset: Option, + input_sources: Vec, + ) -> Result { + // NOTE: These constants must be kept in sync with the TDL package definitions. + const CLP_TDL_PACKAGE_NAME: &str = "clp"; + const COMPRESSION_TASK_FUNC: &str = "compression::clp_s_compress"; + const COMMIT_TASK_FUNC: &str = "compression::clp_s_commit"; + const COMPRESSION_TASK_NUM_INPUTS: usize = 3; + + let commit_task = TerminationTaskDescriptor { + tdl_context: TdlContext { + package: CLP_TDL_PACKAGE_NAME.to_owned(), + task_func: COMMIT_TASK_FUNC.to_owned(), + }, + execution_policy: None, + }; + let mut graph = TaskGraph::new(Some(commit_task), None)?; + + let mut inputs: Vec = + Vec::with_capacity(input_sources.len() * COMPRESSION_TASK_NUM_INPUTS); + for input_source in input_sources { + let timeout_ms: u64 = 5 * 1000 * input_source.object_keys.len() as u64; + let execution_policy = ExecutionPolicy { + max_num_retry: 2, + max_num_instances: 1, + timeout_policy: TimeoutPolicy { + soft_timeout_ms: timeout_ms, + hard_timeout_ms: timeout_ms + 1, + }, + }; + graph.insert_task(TaskDescriptor { + tdl_context: TdlContext { + package: CLP_TDL_PACKAGE_NAME.to_owned(), + task_func: COMPRESSION_TASK_FUNC.to_owned(), + }, + execution_policy: Some(execution_policy), + inputs: vec![ + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( + "ClpSCompressionOption", + )?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( + "Option", + )?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( + "Vec", + )?), + ], + outputs: vec![DataTypeDescriptor::Value( + ValueTypeDescriptor::struct_from_name("Vec")?, + )], + input_sources: None, + })?; + inputs.push(TaskInput::ValuePayload(rmp_serde::to_vec(&clp_s_option)?)); + inputs.push(TaskInput::ValuePayload(rmp_serde::to_vec(&dataset)?)); + inputs.push(TaskInput::ValuePayload(rmp_serde::to_vec(&input_source)?)); + } + + let job_id = self.submit_job(resource_group_id, &graph, inputs).await?; + + tracing::info!( + compression_job_id = % compression_job_id, + spider_job_id = % job_id, + num_tasks = graph.get_num_tasks(), + "Submitted compression job to Spider." + ); + + Ok(job_id) + } + + /// # Errors + /// + /// Returns an error if: + /// + /// * Forwards [`SpiderClient::start_job`]'s return values on failure, except + /// [`ClientError::InvalidJobState`], which indicates the job has already been started. + /// * Forwards [`SpiderClient::get_job_state`]'s return values on failure. + async fn run_s3_compression_job_to_completion( + &self, + spider_job_id: JobId, + initial_poll_backoff: Duration, + max_poll_backoff: Duration, + ) -> Result { + /// The multiplier applied to the poll backoff after each non-terminal poll. + const POLL_BACKOFF_FACTOR: u32 = 2; + + match self.start_job(spider_job_id).await { + Ok(_) | Err(ClientError::InvalidJobState(_)) => {} + Err(error) => return Err(error.into()), + } + + let mut backoff = initial_poll_backoff; + let terminal_state = loop { + let state = self.get_job_state(spider_job_id).await?; + if state.is_terminal() { + break state; + } + tokio::time::sleep(backoff).await; + backoff = (backoff * POLL_BACKOFF_FACTOR).min(max_poll_backoff); + }; + + Ok(match terminal_state { + JobState::Succeeded => CompressionJobOutcome::Succeeded, + JobState::Failed => CompressionJobOutcome::Failed { + error_message: self + .get_job_error(spider_job_id) + .await + .unwrap_or_else(|error| format!("")), + }, + JobState::Cancelled => CompressionJobOutcome::Cancelled, + _ => unreachable!(), + }) + } +} diff --git a/components/compression-coordinator/src/error.rs b/components/compression-coordinator/src/error.rs index 027c335c9c..1a2f65934a 100644 --- a/components/compression-coordinator/src/error.rs +++ b/components/compression-coordinator/src/error.rs @@ -2,4 +2,15 @@ /// Errors returned by the compression coordinator. #[derive(Debug, thiserror::Error)] -pub enum Error {} +pub enum Error { + /// Failed to build or serialize the compression task graph. + #[error("failed to build the compression task graph: {0}")] + TaskGraph(#[from] spider_core::task::Error), + + /// Failed to msgpack-serialize a task input. + #[error("failed to serialize a task input: {0}")] + TaskInputSerialization(#[from] rmp_serde::encode::Error), + + #[error("spider request failure: {0}")] + SpiderClient(#[from] spider_client::error::ClientError), +} From c7d1277f001c003788ab2afc27e407142ef4ac29 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 10:05:05 -0400 Subject: [PATCH 05/16] Update. --- .../src/compression_job_submitter/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/mod.rs b/components/compression-coordinator/src/compression_job_submitter/mod.rs index 07fb09a871..aee0c5d2dc 100644 --- a/components/compression-coordinator/src/compression_job_submitter/mod.rs +++ b/components/compression-coordinator/src/compression_job_submitter/mod.rs @@ -11,7 +11,10 @@ use clp_rust_utils::{ task_io::compression::{ClpSCompressionOption, S3InputSource}, }; use serde::{Deserialize, Serialize}; -use spider_core::types::id::{JobId, ResourceGroupId}; +use spider_core::{ + task::TimeoutPolicy, + types::id::{JobId, ResourceGroupId}, +}; use crate::error::Error; @@ -40,8 +43,9 @@ pub trait S3CompressionJobSubmitter: Clone + Send + Sync { /// * `resource_group_id` - The Spider resource group to register the job under. /// * `clp_s_option` - `clp-s` tuning options shared by every task in the job. /// * `dataset` - The dataset to compress into. - /// * `input_sources` - S3 input sources to compress. Each represents an input to a compression - /// task. + /// * `input_sources` - S3 input sources to compress, each paired with the timeout policy to + /// apply to the compression task it creates. Each element represents an input to a + /// compression task. /// /// # Returns /// @@ -56,7 +60,7 @@ pub trait S3CompressionJobSubmitter: Clone + Send + Sync { resource_group_id: ResourceGroupId, clp_s_option: ClpSCompressionOption, dataset: Option, - input_sources: Vec, + input_sources: Vec<(S3InputSource, TimeoutPolicy)>, ) -> Result; /// Idempotently starts the job identified by `spider_job_id` (only if it hasn't already been From ad6a3f77748460c08a4e58bcee1dc4d7a219a1ed Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 10:15:10 -0400 Subject: [PATCH 06/16] Update. Done. --- .../src/compression_job_submitter/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/mod.rs b/components/compression-coordinator/src/compression_job_submitter/mod.rs index aee0c5d2dc..012e3f11c4 100644 --- a/components/compression-coordinator/src/compression_job_submitter/mod.rs +++ b/components/compression-coordinator/src/compression_job_submitter/mod.rs @@ -12,7 +12,7 @@ use clp_rust_utils::{ }; use serde::{Deserialize, Serialize}; use spider_core::{ - task::TimeoutPolicy, + task::ExecutionPolicy, types::id::{JobId, ResourceGroupId}, }; @@ -43,9 +43,10 @@ pub trait S3CompressionJobSubmitter: Clone + Send + Sync { /// * `resource_group_id` - The Spider resource group to register the job under. /// * `clp_s_option` - `clp-s` tuning options shared by every task in the job. /// * `dataset` - The dataset to compress into. - /// * `input_sources` - S3 input sources to compress, each paired with the timeout policy to + /// * `input_sources` - S3 input sources to compress, each paired with the execution policy to /// apply to the compression task it creates. Each element represents an input to a /// compression task. + /// * `commit_task_execution_policy` - The execution policy to apply to the job's commit task. /// /// # Returns /// @@ -60,7 +61,8 @@ pub trait S3CompressionJobSubmitter: Clone + Send + Sync { resource_group_id: ResourceGroupId, clp_s_option: ClpSCompressionOption, dataset: Option, - input_sources: Vec<(S3InputSource, TimeoutPolicy)>, + input_sources: Vec<(S3InputSource, ExecutionPolicy)>, + commit_task_execution_policy: ExecutionPolicy, ) -> Result; /// Idempotently starts the job identified by `spider_job_id` (only if it hasn't already been From 2323707abe3b8bb020453f21924932c6a2737bc6 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 10:21:43 -0400 Subject: [PATCH 07/16] Done. --- .../src/compression_job_submitter/spider.rs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index 88d0aa5717..03ba1c3a47 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -17,7 +17,6 @@ use spider_core::{ TaskGraph, TdlContext, TerminationTaskDescriptor, - TimeoutPolicy, ValueTypeDescriptor, }, types::{ @@ -48,7 +47,8 @@ impl S3CompressionJobSubmitter for SpiderClient { resource_group_id: ResourceGroupId, clp_s_option: ClpSCompressionOption, dataset: Option, - input_sources: Vec, + input_sources: Vec<(S3InputSource, ExecutionPolicy)>, + commit_task_execution_policy: ExecutionPolicy, ) -> Result { // NOTE: These constants must be kept in sync with the TDL package definitions. const CLP_TDL_PACKAGE_NAME: &str = "clp"; @@ -61,22 +61,13 @@ impl S3CompressionJobSubmitter for SpiderClient { package: CLP_TDL_PACKAGE_NAME.to_owned(), task_func: COMMIT_TASK_FUNC.to_owned(), }, - execution_policy: None, + execution_policy: Some(commit_task_execution_policy), }; let mut graph = TaskGraph::new(Some(commit_task), None)?; let mut inputs: Vec = Vec::with_capacity(input_sources.len() * COMPRESSION_TASK_NUM_INPUTS); - for input_source in input_sources { - let timeout_ms: u64 = 5 * 1000 * input_source.object_keys.len() as u64; - let execution_policy = ExecutionPolicy { - max_num_retry: 2, - max_num_instances: 1, - timeout_policy: TimeoutPolicy { - soft_timeout_ms: timeout_ms, - hard_timeout_ms: timeout_ms + 1, - }, - }; + for (input_source, execution_policy) in input_sources { graph.insert_task(TaskDescriptor { tdl_context: TdlContext { package: CLP_TDL_PACKAGE_NAME.to_owned(), From 7f7562db7c71d83eb17c74126e7b443074c61700 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 20 Jul 2026 10:30:05 -0400 Subject: [PATCH 08/16] Add partition --- Cargo.lock | 8 + Cargo.toml | 1 + components/clp-rust-utils/src/lib.rs | 1 + components/clp-rust-utils/src/task_io.rs | 1 + .../clp-rust-utils/src/task_io/compression.rs | 41 +++ components/compression-coordinator/Cargo.toml | 12 + components/compression-coordinator/src/lib.rs | 3 + .../compression-coordinator/src/partition.rs | 248 ++++++++++++++++++ 8 files changed, 315 insertions(+) create mode 100644 components/clp-rust-utils/src/task_io.rs create mode 100644 components/clp-rust-utils/src/task_io/compression.rs create mode 100644 components/compression-coordinator/Cargo.toml create mode 100644 components/compression-coordinator/src/lib.rs create mode 100644 components/compression-coordinator/src/partition.rs diff --git a/Cargo.lock b/Cargo.lock index 46840eb622..0c160586d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -988,6 +988,14 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "compression-coordinator" +version = "0.13.1-dev" +dependencies = [ + "clp-rust-utils", + "strsim", +] + [[package]] name = "concurrent-queue" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index a3a8bef443..c8f8676876 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "components/api-server", "components/clp-rust-utils", + "components/compression-coordinator", "components/log-ingestor" ] resolver = "3" diff --git a/components/clp-rust-utils/src/lib.rs b/components/clp-rust-utils/src/lib.rs index b3e0d6af7d..e6091b96f4 100644 --- a/components/clp-rust-utils/src/lib.rs +++ b/components/clp-rust-utils/src/lib.rs @@ -8,6 +8,7 @@ pub mod logging; pub mod s3; pub mod serde; pub mod sqs; +pub mod task_io; pub mod telemetry; pub mod types; diff --git a/components/clp-rust-utils/src/task_io.rs b/components/clp-rust-utils/src/task_io.rs new file mode 100644 index 0000000000..376ef623d5 --- /dev/null +++ b/components/clp-rust-utils/src/task_io.rs @@ -0,0 +1 @@ +pub mod compression; diff --git a/components/clp-rust-utils/src/task_io/compression.rs b/components/clp-rust-utils/src/task_io/compression.rs new file mode 100644 index 0000000000..db6c5ca627 --- /dev/null +++ b/components/clp-rust-utils/src/task_io/compression.rs @@ -0,0 +1,41 @@ +//! Protocol types exchanged with the Spider (Huntsman) tasks that run CLP S3 compression jobs. + +use non_empty_string::NonEmptyString; +use serde::{Deserialize, Serialize}; + +use crate::clp_config::AwsAuthentication; + +/// `clp-s` tuning and engine options for a compression job. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClpSCompressionOption { + pub target_encoded_size: u64, + pub compression_level: i32, + pub timestamp_key: Option, +} + +/// Input source for a compression task. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct S3InputSource { + pub endpoint_url: Option, + pub region_code: Option, + pub bucket: NonEmptyString, + pub aws_authentication: AwsAuthentication, + pub object_keys: Vec, +} + +/// Output of a compression task. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CompressionTaskOutput { + pub dataset: Option, + pub archives: Vec, +} + +/// Metadata of an archive produced by `clp-s`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArchiveMetadata { + pub id: String, + pub begin_timestamp: i64, + pub end_timestamp: i64, + pub size: i64, + pub uncompressed_size: i64, +} diff --git a/components/compression-coordinator/Cargo.toml b/components/compression-coordinator/Cargo.toml new file mode 100644 index 0000000000..529018b543 --- /dev/null +++ b/components/compression-coordinator/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "compression-coordinator" +version = "0.13.1-dev" +edition = "2024" + +[lib] +name = "compression_coordinator" +path = "src/lib.rs" + +[dependencies] +clp-rust-utils = { path = "../clp-rust-utils" } +strsim = "0.11.1" diff --git a/components/compression-coordinator/src/lib.rs b/components/compression-coordinator/src/lib.rs new file mode 100644 index 0000000000..db7cb721dd --- /dev/null +++ b/components/compression-coordinator/src/lib.rs @@ -0,0 +1,3 @@ +//! S3 input partitioning for compression jobs. + +pub mod partition; diff --git a/components/compression-coordinator/src/partition.rs b/components/compression-coordinator/src/partition.rs new file mode 100644 index 0000000000..c2ef207eb0 --- /dev/null +++ b/components/compression-coordinator/src/partition.rs @@ -0,0 +1,248 @@ +//! Partitioning of S3 objects into compression-task inputs. + +use std::{collections::VecDeque, path::Path}; + +use clp_rust_utils::{ + clp_config::S3Config, + job_config::{ClpIoConfig, InputConfig}, + s3::ObjectMetadata, + task_io::compression::S3InputSource, +}; + +/// Buffers S3 object metadata and partitions the objects into compression-task inputs. +pub struct PathsToCompressBuffer { + files: Vec, + tasks_input_sources: Vec, + total_file_size: u64, + target_archive_size: u64, + file_size_to_trigger_compression: u64, + s3_config: S3Config, +} + +impl PathsToCompressBuffer { + /// Creates an empty buffer using the compression target and S3 settings in `clp_io_config`. + /// + /// # Returns + /// + /// A new empty [`PathsToCompressBuffer`]. + #[must_use] + pub fn new(clp_io_config: &ClpIoConfig) -> Self { + let target_archive_size = clp_io_config.output.target_archive_size; + let s3_config = match &clp_io_config.input { + InputConfig::S3InputConfig { config } => config.s3_config.clone(), + InputConfig::S3ObjectMetadataInputConfig { config } => config.s3_config.clone(), + }; + + Self { + files: Vec::new(), + tasks_input_sources: Vec::new(), + total_file_size: 0, + target_archive_size, + file_size_to_trigger_compression: target_archive_size * 2, + s3_config, + } + } + + /// Adds an S3 object to the buffer and partitions buffered objects when the trigger size is + /// reached. + pub fn add_file(&mut self, metadata: ObjectMetadata) { + self.total_file_size += estimate_uncompressed_size(&metadata); + self.files.push(metadata); + if self.total_file_size >= self.file_size_to_trigger_compression { + self.partition_and_compress(false); + } + } + + /// Consumes the buffer and returns the compression-task inputs created from completed + /// partitions. + /// + /// # Returns + /// + /// One [`S3InputSource`] per completed partition. + #[must_use] + pub fn get_tasks_input_sources(self) -> Vec { + self.tasks_input_sources + } + + /// Partitions every buffered object, including a final partition smaller than the target size. + pub fn flush(&mut self) { + self.partition_and_compress(true); + } + + /// Converts a non-empty partition into a compression-task input and stores it for later + /// submission. + fn submit_partition_for_compression(&mut self, files: Vec) { + if files.is_empty() { + return; + } + + let object_keys = files + .into_iter() + .map(|metadata| String::from(metadata.key)) + .collect(); + + self.tasks_input_sources.push(S3InputSource { + endpoint_url: self.s3_config.endpoint_url.clone(), + region_code: self.s3_config.region_code.clone(), + bucket: self.s3_config.bucket.clone(), + aws_authentication: self.s3_config.aws_authentication.clone(), + object_keys, + }); + } + + /// Partitions buffered objects into target-sized compression-task inputs. + /// + /// When `flush_buffer` is false, objects that cannot fill another target-sized partition + /// remain buffered. Otherwise, the final partial partition is also emitted. + fn partition_and_compress(&mut self, flush_buffer: bool) { + if !flush_buffer && self.total_file_size < self.target_archive_size { + return; + } + if self.files.is_empty() { + return; + } + + let mut round_robin_partition = RoundRobinPartition::new(std::mem::take(&mut self.files)); + + let mut has_more_files = true; + while has_more_files && (flush_buffer || self.total_file_size >= self.target_archive_size) { + let mut partition = Vec::new(); + let mut partition_size = 0; + + while partition_size < self.target_archive_size { + let Some(file) = round_robin_partition.next() else { + has_more_files = false; + break; + }; + + partition_size += estimate_uncompressed_size(&file); + partition.push(file); + } + + self.submit_partition_for_compression(partition); + self.total_file_size -= partition_size; + } + + self.files = round_robin_partition.into_remaining_files(); + } +} + +/// Owns filename-similarity groups and returns one object from each group in round-robin order. +struct RoundRobinPartition { + groups: VecDeque>, +} + +impl RoundRobinPartition { + /// Groups objects by filename similarity and creates a round-robin iterator over the groups. + /// + /// # Returns + /// + /// A new [`RoundRobinPartition`]. + fn new(files: Vec) -> Self { + let groups = group_files_by_similar_filenames(files) + .into_iter() + .filter(|group| !group.is_empty()) + .map(IntoIterator::into_iter) + .collect(); + + Self { groups } + } + + /// Consumes the iterator and collects every object that has not yet been returned. + /// + /// # Returns + /// + /// The remaining objects, flattened according to the iterator's current group order. + fn into_remaining_files(self) -> Vec { + self.groups.into_iter().flatten().collect() + } +} + +impl Iterator for RoundRobinPartition { + type Item = ObjectMetadata; + + fn next(&mut self) -> Option { + let mut group = self.groups.pop_front()?; + let file = group.next()?; + if !group.as_slice().is_empty() { + self.groups.push_back(group); + } + Some(file) + } +} + +/// Estimates an S3 object's uncompressed size from its compressor suffix. +/// +/// # Returns +/// +/// The estimated uncompressed size. +fn estimate_uncompressed_size(metadata: &ObjectMetadata) -> u64 { + const GZIP_COMPRESSION_RATIO_ESTIMATE: u64 = 13; + const GZIP_SUFFIXES: &[&str] = &[".gz", ".gzip", ".tgz", ".tar.gz"]; + const ZSTD_COMPRESSION_RATIO_ESTIMATE: u64 = 8; + const ZSTD_SUFFIXES: &[&str] = &[".zstd", ".zstandard", ".tar.zstd", ".tar.zstandard"]; + + let key = metadata.key.as_str(); + if GZIP_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) { + metadata.size * GZIP_COMPRESSION_RATIO_ESTIMATE + } else if ZSTD_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) { + metadata.size * ZSTD_COMPRESSION_RATIO_ESTIMATE + } else { + metadata.size + } +} + +/// Gets the filename portion of an S3 object's key. +/// +/// # Returns +/// +/// The key's filename, or the complete key if it has no filename component. +fn filename(metadata: &ObjectMetadata) -> &str { + let key = metadata.key.as_str(); + let Some(file_name) = Path::new(key) + .file_name() + .and_then(|file_name| file_name.to_str()) + else { + return key; + }; + file_name +} + +/// Groups S3 objects by filename similarity. +/// +/// Places files with sufficiently similar filenames into the same group based +/// on their normalized Levenshtein similarity. +/// +/// # Returns +/// +/// A vector of filename-similarity groups, or an empty vector if files is +/// empty. +fn group_files_by_similar_filenames(mut files: Vec) -> Vec> { + const FILE_GROUPING_MIN_LEVENSHTEIN_RATIO: f64 = 0.6; + + files.sort_by(|a, b| filename(a).cmp(filename(b))); + + let mut files = files.into_iter(); + let Some(first_file) = files.next() else { + return Vec::new(); + }; + + let mut previous_filename = filename(&first_file).to_owned(); + let mut current_group = vec![first_file]; + let mut groups = Vec::new(); + + for file in files { + let current_filename = filename(&file).to_owned(); + if strsim::normalized_levenshtein(&previous_filename, ¤t_filename) + < FILE_GROUPING_MIN_LEVENSHTEIN_RATIO + { + groups.push(current_group); + current_group = Vec::new(); + } + current_group.push(file); + previous_filename = current_filename; + } + groups.push(current_group); + + groups +} From bf903aa9a3449bba0ecf3dcf2418d90b4b1b3bb8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 20 Jul 2026 10:30:05 -0400 Subject: [PATCH 09/16] Add partition --- Cargo.lock | 1 + components/compression-coordinator/Cargo.toml | 1 + components/compression-coordinator/src/lib.rs | 1 + .../compression-coordinator/src/partition.rs | 248 ++++++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 components/compression-coordinator/src/partition.rs diff --git a/Cargo.lock b/Cargo.lock index 1eba6ec59d..8bd9e84d60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -996,6 +996,7 @@ dependencies = [ "clp-rust-utils", "serde", "spider-core", + "strsim", "thiserror", ] diff --git a/components/compression-coordinator/Cargo.toml b/components/compression-coordinator/Cargo.toml index 861b596414..9fa64c72a7 100644 --- a/components/compression-coordinator/Cargo.toml +++ b/components/compression-coordinator/Cargo.toml @@ -12,4 +12,5 @@ async-trait = "0.1.89" clp-rust-utils = { path = "../clp-rust-utils" } serde = { version = "1.0.228", features = ["derive"] } spider-core = { git = "https://github.com/y-scope/spider.git", branch = "main" } +strsim = "0.11.1" thiserror = "2.0.18" diff --git a/components/compression-coordinator/src/lib.rs b/components/compression-coordinator/src/lib.rs index 47921c0041..ca21c72c24 100644 --- a/components/compression-coordinator/src/lib.rs +++ b/components/compression-coordinator/src/lib.rs @@ -3,5 +3,6 @@ pub mod compression_job_submitter; mod error; +pub mod partition; pub use error::Error; diff --git a/components/compression-coordinator/src/partition.rs b/components/compression-coordinator/src/partition.rs new file mode 100644 index 0000000000..c2ef207eb0 --- /dev/null +++ b/components/compression-coordinator/src/partition.rs @@ -0,0 +1,248 @@ +//! Partitioning of S3 objects into compression-task inputs. + +use std::{collections::VecDeque, path::Path}; + +use clp_rust_utils::{ + clp_config::S3Config, + job_config::{ClpIoConfig, InputConfig}, + s3::ObjectMetadata, + task_io::compression::S3InputSource, +}; + +/// Buffers S3 object metadata and partitions the objects into compression-task inputs. +pub struct PathsToCompressBuffer { + files: Vec, + tasks_input_sources: Vec, + total_file_size: u64, + target_archive_size: u64, + file_size_to_trigger_compression: u64, + s3_config: S3Config, +} + +impl PathsToCompressBuffer { + /// Creates an empty buffer using the compression target and S3 settings in `clp_io_config`. + /// + /// # Returns + /// + /// A new empty [`PathsToCompressBuffer`]. + #[must_use] + pub fn new(clp_io_config: &ClpIoConfig) -> Self { + let target_archive_size = clp_io_config.output.target_archive_size; + let s3_config = match &clp_io_config.input { + InputConfig::S3InputConfig { config } => config.s3_config.clone(), + InputConfig::S3ObjectMetadataInputConfig { config } => config.s3_config.clone(), + }; + + Self { + files: Vec::new(), + tasks_input_sources: Vec::new(), + total_file_size: 0, + target_archive_size, + file_size_to_trigger_compression: target_archive_size * 2, + s3_config, + } + } + + /// Adds an S3 object to the buffer and partitions buffered objects when the trigger size is + /// reached. + pub fn add_file(&mut self, metadata: ObjectMetadata) { + self.total_file_size += estimate_uncompressed_size(&metadata); + self.files.push(metadata); + if self.total_file_size >= self.file_size_to_trigger_compression { + self.partition_and_compress(false); + } + } + + /// Consumes the buffer and returns the compression-task inputs created from completed + /// partitions. + /// + /// # Returns + /// + /// One [`S3InputSource`] per completed partition. + #[must_use] + pub fn get_tasks_input_sources(self) -> Vec { + self.tasks_input_sources + } + + /// Partitions every buffered object, including a final partition smaller than the target size. + pub fn flush(&mut self) { + self.partition_and_compress(true); + } + + /// Converts a non-empty partition into a compression-task input and stores it for later + /// submission. + fn submit_partition_for_compression(&mut self, files: Vec) { + if files.is_empty() { + return; + } + + let object_keys = files + .into_iter() + .map(|metadata| String::from(metadata.key)) + .collect(); + + self.tasks_input_sources.push(S3InputSource { + endpoint_url: self.s3_config.endpoint_url.clone(), + region_code: self.s3_config.region_code.clone(), + bucket: self.s3_config.bucket.clone(), + aws_authentication: self.s3_config.aws_authentication.clone(), + object_keys, + }); + } + + /// Partitions buffered objects into target-sized compression-task inputs. + /// + /// When `flush_buffer` is false, objects that cannot fill another target-sized partition + /// remain buffered. Otherwise, the final partial partition is also emitted. + fn partition_and_compress(&mut self, flush_buffer: bool) { + if !flush_buffer && self.total_file_size < self.target_archive_size { + return; + } + if self.files.is_empty() { + return; + } + + let mut round_robin_partition = RoundRobinPartition::new(std::mem::take(&mut self.files)); + + let mut has_more_files = true; + while has_more_files && (flush_buffer || self.total_file_size >= self.target_archive_size) { + let mut partition = Vec::new(); + let mut partition_size = 0; + + while partition_size < self.target_archive_size { + let Some(file) = round_robin_partition.next() else { + has_more_files = false; + break; + }; + + partition_size += estimate_uncompressed_size(&file); + partition.push(file); + } + + self.submit_partition_for_compression(partition); + self.total_file_size -= partition_size; + } + + self.files = round_robin_partition.into_remaining_files(); + } +} + +/// Owns filename-similarity groups and returns one object from each group in round-robin order. +struct RoundRobinPartition { + groups: VecDeque>, +} + +impl RoundRobinPartition { + /// Groups objects by filename similarity and creates a round-robin iterator over the groups. + /// + /// # Returns + /// + /// A new [`RoundRobinPartition`]. + fn new(files: Vec) -> Self { + let groups = group_files_by_similar_filenames(files) + .into_iter() + .filter(|group| !group.is_empty()) + .map(IntoIterator::into_iter) + .collect(); + + Self { groups } + } + + /// Consumes the iterator and collects every object that has not yet been returned. + /// + /// # Returns + /// + /// The remaining objects, flattened according to the iterator's current group order. + fn into_remaining_files(self) -> Vec { + self.groups.into_iter().flatten().collect() + } +} + +impl Iterator for RoundRobinPartition { + type Item = ObjectMetadata; + + fn next(&mut self) -> Option { + let mut group = self.groups.pop_front()?; + let file = group.next()?; + if !group.as_slice().is_empty() { + self.groups.push_back(group); + } + Some(file) + } +} + +/// Estimates an S3 object's uncompressed size from its compressor suffix. +/// +/// # Returns +/// +/// The estimated uncompressed size. +fn estimate_uncompressed_size(metadata: &ObjectMetadata) -> u64 { + const GZIP_COMPRESSION_RATIO_ESTIMATE: u64 = 13; + const GZIP_SUFFIXES: &[&str] = &[".gz", ".gzip", ".tgz", ".tar.gz"]; + const ZSTD_COMPRESSION_RATIO_ESTIMATE: u64 = 8; + const ZSTD_SUFFIXES: &[&str] = &[".zstd", ".zstandard", ".tar.zstd", ".tar.zstandard"]; + + let key = metadata.key.as_str(); + if GZIP_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) { + metadata.size * GZIP_COMPRESSION_RATIO_ESTIMATE + } else if ZSTD_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) { + metadata.size * ZSTD_COMPRESSION_RATIO_ESTIMATE + } else { + metadata.size + } +} + +/// Gets the filename portion of an S3 object's key. +/// +/// # Returns +/// +/// The key's filename, or the complete key if it has no filename component. +fn filename(metadata: &ObjectMetadata) -> &str { + let key = metadata.key.as_str(); + let Some(file_name) = Path::new(key) + .file_name() + .and_then(|file_name| file_name.to_str()) + else { + return key; + }; + file_name +} + +/// Groups S3 objects by filename similarity. +/// +/// Places files with sufficiently similar filenames into the same group based +/// on their normalized Levenshtein similarity. +/// +/// # Returns +/// +/// A vector of filename-similarity groups, or an empty vector if files is +/// empty. +fn group_files_by_similar_filenames(mut files: Vec) -> Vec> { + const FILE_GROUPING_MIN_LEVENSHTEIN_RATIO: f64 = 0.6; + + files.sort_by(|a, b| filename(a).cmp(filename(b))); + + let mut files = files.into_iter(); + let Some(first_file) = files.next() else { + return Vec::new(); + }; + + let mut previous_filename = filename(&first_file).to_owned(); + let mut current_group = vec![first_file]; + let mut groups = Vec::new(); + + for file in files { + let current_filename = filename(&file).to_owned(); + if strsim::normalized_levenshtein(&previous_filename, ¤t_filename) + < FILE_GROUPING_MIN_LEVENSHTEIN_RATIO + { + groups.push(current_group); + current_group = Vec::new(); + } + current_group.push(file); + previous_filename = current_filename; + } + groups.push(current_group); + + groups +} From 30105a5260b87372b48e82ff127b70040b38fdcb Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 20 Jul 2026 17:58:24 -0400 Subject: [PATCH 10/16] Apply Zhihao's review changes. --- Cargo.lock | 1 + components/compression-coordinator/Cargo.toml | 3 + .../compression-coordinator/src/error.rs | 5 +- .../compression-coordinator/src/partition.rs | 379 +++++++++++++----- 4 files changed, 295 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8bd9e84d60..cadc8de79b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,6 +994,7 @@ version = "0.13.1-dev" dependencies = [ "async-trait", "clp-rust-utils", + "non-empty-string", "serde", "spider-core", "strsim", diff --git a/components/compression-coordinator/Cargo.toml b/components/compression-coordinator/Cargo.toml index 9fa64c72a7..0140fe7916 100644 --- a/components/compression-coordinator/Cargo.toml +++ b/components/compression-coordinator/Cargo.toml @@ -14,3 +14,6 @@ serde = { version = "1.0.228", features = ["derive"] } spider-core = { git = "https://github.com/y-scope/spider.git", branch = "main" } strsim = "0.11.1" thiserror = "2.0.18" + +[dev-dependencies] +non-empty-string = "0.2.6" diff --git a/components/compression-coordinator/src/error.rs b/components/compression-coordinator/src/error.rs index 027c335c9c..0a88e73188 100644 --- a/components/compression-coordinator/src/error.rs +++ b/components/compression-coordinator/src/error.rs @@ -2,4 +2,7 @@ /// Errors returned by the compression coordinator. #[derive(Debug, thiserror::Error)] -pub enum Error {} +pub enum Error { + #[error("S3 bucket mismatch: expected `{0}`, but got `{1}`")] + S3BucketMismatch(String, String), +} diff --git a/components/compression-coordinator/src/partition.rs b/components/compression-coordinator/src/partition.rs index c2ef207eb0..907ab9fc90 100644 --- a/components/compression-coordinator/src/partition.rs +++ b/components/compression-coordinator/src/partition.rs @@ -9,22 +9,27 @@ use clp_rust_utils::{ task_io::compression::S3InputSource, }; -/// Buffers S3 object metadata and partitions the objects into compression-task inputs. -pub struct PathsToCompressBuffer { - files: Vec, - tasks_input_sources: Vec, - total_file_size: u64, +use crate::Error; + +/// Builds inputs for compression tasks. +/// +/// The builder buffers input objects internally and groups them into partitions. Each partition +/// contains the inputs for a single compression task. +pub struct CompressionInputBuilder { + buffer: Vec, + partitioned_task_inputs: Vec, + total_buffered_size: u64, target_archive_size: u64, - file_size_to_trigger_compression: u64, + buffer_size_to_trigger_partition: u64, s3_config: S3Config, } -impl PathsToCompressBuffer { - /// Creates an empty buffer using the compression target and S3 settings in `clp_io_config`. +impl CompressionInputBuilder { + /// Factory function. /// /// # Returns /// - /// A new empty [`PathsToCompressBuffer`]. + /// A newly created [`CompressionInputBuilder`] with an empty buffer. #[must_use] pub fn new(clp_io_config: &ClpIoConfig) -> Self { let target_archive_size = clp_io_config.output.target_archive_size; @@ -34,54 +39,67 @@ impl PathsToCompressBuffer { }; Self { - files: Vec::new(), - tasks_input_sources: Vec::new(), - total_file_size: 0, + buffer: Vec::new(), + partitioned_task_inputs: Vec::new(), + total_buffered_size: 0, target_archive_size, - file_size_to_trigger_compression: target_archive_size * 2, + buffer_size_to_trigger_partition: target_archive_size * 2, s3_config, } } - /// Adds an S3 object to the buffer and partitions buffered objects when the trigger size is - /// reached. - pub fn add_file(&mut self, metadata: ObjectMetadata) { - self.total_file_size += estimate_uncompressed_size(&metadata); - self.files.push(metadata); - if self.total_file_size >= self.file_size_to_trigger_compression { - self.partition_and_compress(false); + /// Adds an S3 object to the buffer. + /// + /// # Errors + /// + /// Returns an error if: + /// + /// * [`Error::S3BucketMismatch`] if `metadata`'s bucket differs from the buffer's configured + /// bucket. + pub fn add(&mut self, metadata: ObjectMetadata) -> Result<(), Error> { + if metadata.bucket != self.s3_config.bucket { + return Err(Error::S3BucketMismatch( + String::from(self.s3_config.bucket.clone()), + String::from(metadata.bucket), + )); + } + + let file = FileMetadata::new(String::from(metadata.key), metadata.size); + self.total_buffered_size += file.estimated_size; + self.buffer.push(file); + if self.total_buffered_size >= self.buffer_size_to_trigger_partition { + self.partition(false); } + + Ok(()) } - /// Consumes the buffer and returns the compression-task inputs created from completed - /// partitions. + /// Flushes any remaining buffered objects and consumes the builder, returning the completed + /// compression task inputs. /// /// # Returns /// - /// One [`S3InputSource`] per completed partition. + /// A vector of input sources, where each source contains the inputs for a single compression + /// task. #[must_use] - pub fn get_tasks_input_sources(self) -> Vec { - self.tasks_input_sources + pub fn into_task_input_sources(mut self) -> Vec { + self.flush(); + self.partitioned_task_inputs } - /// Partitions every buffered object, including a final partition smaller than the target size. - pub fn flush(&mut self) { - self.partition_and_compress(true); + /// Partitions all the objects in the current buffer. + fn flush(&mut self) { + self.partition(true); } - /// Converts a non-empty partition into a compression-task input and stores it for later - /// submission. - fn submit_partition_for_compression(&mut self, files: Vec) { - if files.is_empty() { + /// Converts a non-empty partition into a compression task input and pushes it to the underlying + /// task inputs buffer. + fn push_input_source(&mut self, object_keys: Vec) { + if object_keys.is_empty() { return; } - let object_keys = files - .into_iter() - .map(|metadata| String::from(metadata.key)) - .collect(); - - self.tasks_input_sources.push(S3InputSource { + self.partitioned_task_inputs.push(S3InputSource { endpoint_url: self.s3_config.endpoint_url.clone(), region_code: self.s3_config.region_code.clone(), bucket: self.s3_config.bucket.clone(), @@ -90,55 +108,76 @@ impl PathsToCompressBuffer { }); } - /// Partitions buffered objects into target-sized compression-task inputs. + /// Partitions buffered objects into target-sized compression task inputs. /// /// When `flush_buffer` is false, objects that cannot fill another target-sized partition /// remain buffered. Otherwise, the final partial partition is also emitted. - fn partition_and_compress(&mut self, flush_buffer: bool) { - if !flush_buffer && self.total_file_size < self.target_archive_size { + fn partition(&mut self, flush_buffer: bool) { + if !flush_buffer && self.total_buffered_size < self.target_archive_size { return; } - if self.files.is_empty() { + if self.buffer.is_empty() { return; } - let mut round_robin_partition = RoundRobinPartition::new(std::mem::take(&mut self.files)); + let mut rr_iterator = RoundRobinIterator::new(std::mem::take(&mut self.buffer)); - let mut has_more_files = true; - while has_more_files && (flush_buffer || self.total_file_size >= self.target_archive_size) { + 'partitioning: while flush_buffer || self.total_buffered_size >= self.target_archive_size { let mut partition = Vec::new(); let mut partition_size = 0; while partition_size < self.target_archive_size { - let Some(file) = round_robin_partition.next() else { - has_more_files = false; - break; + let Some(file) = rr_iterator.next() else { + self.push_input_source(partition); + break 'partitioning; }; - partition_size += estimate_uncompressed_size(&file); - partition.push(file); + partition_size += file.estimated_size; + self.total_buffered_size -= file.estimated_size; + partition.push(file.path); } - self.submit_partition_for_compression(partition); - self.total_file_size -= partition_size; + self.push_input_source(partition); } - self.files = round_robin_partition.into_remaining_files(); + self.buffer = rr_iterator.into_flatten(); + } +} + +/// An S3 object's key and its estimated uncompressed size. +struct FileMetadata { + path: String, + estimated_size: u64, +} + +impl FileMetadata { + /// Creates metadata for the object identified by `path`, estimating its uncompressed size from + /// its raw `size`. + /// + /// # Returns + /// + /// A new [`FileMetadata`]. + fn new(path: String, size: u64) -> Self { + let estimated_size = estimate_uncompressed_size(&path, size); + Self { + path, + estimated_size, + } } } /// Owns filename-similarity groups and returns one object from each group in round-robin order. -struct RoundRobinPartition { - groups: VecDeque>, +struct RoundRobinIterator { + groups: VecDeque>, } -impl RoundRobinPartition { +impl RoundRobinIterator { /// Groups objects by filename similarity and creates a round-robin iterator over the groups. /// /// # Returns /// - /// A new [`RoundRobinPartition`]. - fn new(files: Vec) -> Self { + /// A new [`RoundRobinIterator`]. + fn new(files: Vec) -> Self { let groups = group_files_by_similar_filenames(files) .into_iter() .filter(|group| !group.is_empty()) @@ -153,13 +192,13 @@ impl RoundRobinPartition { /// # Returns /// /// The remaining objects, flattened according to the iterator's current group order. - fn into_remaining_files(self) -> Vec { + fn into_flatten(self) -> Vec { self.groups.into_iter().flatten().collect() } } -impl Iterator for RoundRobinPartition { - type Item = ObjectMetadata; +impl Iterator for RoundRobinIterator { + type Item = FileMetadata; fn next(&mut self) -> Option { let mut group = self.groups.pop_front()?; @@ -176,19 +215,18 @@ impl Iterator for RoundRobinPartition { /// # Returns /// /// The estimated uncompressed size. -fn estimate_uncompressed_size(metadata: &ObjectMetadata) -> u64 { +fn estimate_uncompressed_size(key: &str, size: u64) -> u64 { const GZIP_COMPRESSION_RATIO_ESTIMATE: u64 = 13; const GZIP_SUFFIXES: &[&str] = &[".gz", ".gzip", ".tgz", ".tar.gz"]; const ZSTD_COMPRESSION_RATIO_ESTIMATE: u64 = 8; const ZSTD_SUFFIXES: &[&str] = &[".zstd", ".zstandard", ".tar.zstd", ".tar.zstandard"]; - let key = metadata.key.as_str(); if GZIP_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) { - metadata.size * GZIP_COMPRESSION_RATIO_ESTIMATE + size * GZIP_COMPRESSION_RATIO_ESTIMATE } else if ZSTD_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) { - metadata.size * ZSTD_COMPRESSION_RATIO_ESTIMATE + size * ZSTD_COMPRESSION_RATIO_ESTIMATE } else { - metadata.size + size } } @@ -197,8 +235,7 @@ fn estimate_uncompressed_size(metadata: &ObjectMetadata) -> u64 { /// # Returns /// /// The key's filename, or the complete key if it has no filename component. -fn filename(metadata: &ObjectMetadata) -> &str { - let key = metadata.key.as_str(); +fn filename(key: &str) -> &str { let Some(file_name) = Path::new(key) .file_name() .and_then(|file_name| file_name.to_str()) @@ -210,39 +247,197 @@ fn filename(metadata: &ObjectMetadata) -> &str { /// Groups S3 objects by filename similarity. /// -/// Places files with sufficiently similar filenames into the same group based -/// on their normalized Levenshtein similarity. +/// Places files with sufficiently similar filenames into the same group based on their normalized +/// Levenshtein similarity. /// /// # Returns /// -/// A vector of filename-similarity groups, or an empty vector if files is -/// empty. -fn group_files_by_similar_filenames(mut files: Vec) -> Vec> { +/// A vector of filename-similarity groups. +fn group_files_by_similar_filenames(mut files: Vec) -> Vec> { const FILE_GROUPING_MIN_LEVENSHTEIN_RATIO: f64 = 0.6; - files.sort_by(|a, b| filename(a).cmp(filename(b))); + files.sort_by(|a, b| filename(&a.path).cmp(filename(&b.path))); + files + .into_iter() + .fold(Vec::new(), |mut groups: Vec>, curr| { + match groups.last_mut() { + Some(group) + if group.last().is_some_and(|prev| { + strsim::normalized_levenshtein(filename(&prev.path), filename(&curr.path)) + >= FILE_GROUPING_MIN_LEVENSHTEIN_RATIO + }) => + { + group.push(curr); + } + _ => groups.push(vec![curr]), + } + groups + }) +} - let mut files = files.into_iter(); - let Some(first_file) = files.next() else { - return Vec::new(); +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use clp_rust_utils::{ + clp_config::AwsAuthentication, + job_config::{OutputConfig, S3InputConfig}, }; + use non_empty_string::NonEmptyString; + + use super::*; + + const TEST_BUCKET: &str = "test-bucket"; + + fn create_non_empty_string(value: &str) -> NonEmptyString { + NonEmptyString::try_from(value).expect("test string literals are non-empty") + } - let mut previous_filename = filename(&first_file).to_owned(); - let mut current_group = vec![first_file]; - let mut groups = Vec::new(); - - for file in files { - let current_filename = filename(&file).to_owned(); - if strsim::normalized_levenshtein(&previous_filename, ¤t_filename) - < FILE_GROUPING_MIN_LEVENSHTEIN_RATIO - { - groups.push(current_group); - current_group = Vec::new(); + fn create_builder(target_archive_size: u64) -> CompressionInputBuilder { + let s3_config = S3Config { + bucket: create_non_empty_string(TEST_BUCKET), + region_code: None, + key_prefix: create_non_empty_string("logs/"), + endpoint_url: None, + aws_authentication: AwsAuthentication::Default, + }; + let clp_io_config = ClpIoConfig { + input: InputConfig::S3InputConfig { + config: S3InputConfig { + s3_config, + keys: None, + dataset: None, + timestamp_key: None, + unstructured: false, + }, + }, + output: OutputConfig { + target_archive_size, + target_dictionaries_size: 1024, + target_encoded_file_size: 1024, + target_segment_size: 1024, + compression_level: 3, + }, + }; + + CompressionInputBuilder::new(&clp_io_config) + } + + fn add_object( + builder: &mut CompressionInputBuilder, + key_to_size: &mut BTreeMap, + key: &str, + size: u64, + ) { + builder + .add(ObjectMetadata { + bucket: create_non_empty_string(TEST_BUCKET), + key: create_non_empty_string(key), + size, + }) + .expect("object's bucket matches the builder's configured bucket"); + key_to_size.insert(String::from(key), size); + } + + fn assert_partition_invariants( + input_sources: &[S3InputSource], + key_to_size: &BTreeMap, + target_archive_size: u64, + ) { + let mut key_counts: BTreeMap<&str, usize> = BTreeMap::new(); + for key in input_sources.iter().flat_map(|source| &source.object_keys) { + *key_counts.entry(key.as_str()).or_default() += 1; } - current_group.push(file); - previous_filename = current_filename; + for key in key_to_size.keys() { + assert_eq!(key_counts.remove(key.as_str()), Some(1)); + } + assert!(key_counts.is_empty()); + + for source in input_sources { + let total_size: u64 = source + .object_keys + .iter() + .map(|key| key_to_size.get(key).expect("every key was added")) + .sum(); + let last_size = source + .object_keys + .last() + .map(|key| key_to_size.get(key).expect("every key was added")) + .expect("`push_input_source` never pushes an empty partition"); + + // Files are appended only while the partition is still under the target, so removing + // the last-appended file must drop the partition back below the target. + assert!(total_size - last_size < target_archive_size); + } + } + + #[test] + fn test_partition_on_flush() { + const TARGET_ARCHIVE_SIZE: u64 = 1000; + const FILE_SIZE: u64 = 100; + const NUM_FILES: u64 = 15; + + let mut builder = create_builder(TARGET_ARCHIVE_SIZE); + let mut key_to_size = BTreeMap::new(); + for i in 0..NUM_FILES { + add_object( + &mut builder, + &mut key_to_size, + &format!("logs/app.log.{i}"), + FILE_SIZE, + ); + } + + assert!(builder.partitioned_task_inputs.is_empty()); + + let input_sources = builder.into_task_input_sources(); + assert!(!input_sources.is_empty()); + assert_partition_invariants(&input_sources, &key_to_size, TARGET_ARCHIVE_SIZE); } - groups.push(current_group); - groups + #[test] + fn test_partition_on_the_fly() { + const TARGET_ARCHIVE_SIZE: u64 = 1000; + const FILE_STEMS: &[&str] = &["app.log", "database.log", "kernel.log", "nginx.log"]; + const NUM_FILES_PER_STEM: u64 = 60; + + let mut builder = create_builder(TARGET_ARCHIVE_SIZE); + let mut key_to_size = BTreeMap::new(); + for stem in FILE_STEMS { + for i in 0..NUM_FILES_PER_STEM { + add_object( + &mut builder, + &mut key_to_size, + &format!("logs/{stem}.{i}"), + 10 + (i % 7) * 30, + ); + } + } + + assert!(!builder.partitioned_task_inputs.is_empty()); + + let input_sources = builder.into_task_input_sources(); + assert_partition_invariants(&input_sources, &key_to_size, TARGET_ARCHIVE_SIZE); + } + + #[test] + fn test_round_robin_iterator() { + let files = ["app.log", "app.log.1", "app.log.2"] + .into_iter() + .chain(["nginx.log", "nginx.log.1", "nginx.log.2"]) + .map(|key| FileMetadata::new(String::from(key), 1)) + .collect(); + + let mut rr_iterator = RoundRobinIterator::new(files); + let yielded: Vec = rr_iterator.by_ref().take(3).map(|file| file.path).collect(); + assert_eq!(yielded, vec!["app.log", "nginx.log", "app.log.1"]); + + let mut remaining: Vec = rr_iterator + .into_flatten() + .into_iter() + .map(|file| file.path) + .collect(); + remaining.sort(); + assert_eq!(remaining, vec!["app.log.2", "nginx.log.1", "nginx.log.2"]); + } } From 9de36fc8b419c202d0770809a33a757bc9d947f1 Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:53:44 -0400 Subject: [PATCH 11/16] Apply suggestion from @Bill-hbrhbr Co-authored-by: Bingran Hu --- .../src/compression_job_submitter/spider.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index 03ba1c3a47..ac9a397511 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -52,8 +52,8 @@ impl S3CompressionJobSubmitter for SpiderClient { ) -> Result { // NOTE: These constants must be kept in sync with the TDL package definitions. const CLP_TDL_PACKAGE_NAME: &str = "clp"; - const COMPRESSION_TASK_FUNC: &str = "compression::clp_s_compress"; - const COMMIT_TASK_FUNC: &str = "compression::clp_s_commit"; + const COMPRESSION_TASK_FUNC: &str = "compression::clp_s_s3_compress"; + const COMMIT_TASK_FUNC: &str = "compression::commit"; const COMPRESSION_TASK_NUM_INPUTS: usize = 3; let commit_task = TerminationTaskDescriptor { From ab89d91ec9c947d2c8d6367e152505154a9b6bc8 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 21 Jul 2026 13:58:07 -0400 Subject: [PATCH 12/16] Fix Spider type tag. --- .../src/compression_job_submitter/spider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index ac9a397511..f76c2ec507 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -82,7 +82,7 @@ impl S3CompressionJobSubmitter for SpiderClient { "Option", )?), DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - "Vec", + "S3InputSource", )?), ], outputs: vec![DataTypeDescriptor::Value( From 86a9179ba2448fe8ab0b3cda11b4f7611d956287 Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:00:18 -0400 Subject: [PATCH 13/16] Apply suggestions from code review Co-authored-by: Bingran Hu --- .../src/compression_job_submitter/spider.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index f76c2ec507..e6f15e2b5a 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -75,18 +75,18 @@ impl S3CompressionJobSubmitter for SpiderClient { }, execution_policy: Some(execution_policy), inputs: vec![ - DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - "ClpSCompressionOption", - )?), - DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - "Option", - )?), - DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - "S3InputSource", - )?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( + stringify!(ClpSCompressionOption), + )?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( + stringify!(Option), + )?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( + stringify!(S3InputSource), + )?), ], outputs: vec![DataTypeDescriptor::Value( - ValueTypeDescriptor::struct_from_name("Vec")?, + ValueTypeDescriptor::struct_from_name("CompressionTaskOutput")?, )], input_sources: None, })?; @@ -128,14 +128,14 @@ impl S3CompressionJobSubmitter for SpiderClient { Err(error) => return Err(error.into()), } - let mut backoff = initial_poll_backoff; + let mut backoff = initial_poll_backoff.min(max_poll_backoff); let terminal_state = loop { let state = self.get_job_state(spider_job_id).await?; if state.is_terminal() { break state; } tokio::time::sleep(backoff).await; - backoff = (backoff * POLL_BACKOFF_FACTOR).min(max_poll_backoff); + backoff = backoff.saturating_mul(POLL_BACKOFF_FACTOR).min(max_poll_backoff); }; Ok(match terminal_state { From ec55097e8421687324636affba6c10862a6f4310 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 21 Jul 2026 17:05:30 -0400 Subject: [PATCH 14/16] last fix --- .../src/compression_job_submitter/spider.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index e6f15e2b5a..fa33c97473 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -5,7 +5,7 @@ use std::time::Duration; use async_trait::async_trait; use clp_rust_utils::{ job_config::CompressionJobId, - task_io::compression::{ClpSCompressionOption, S3InputSource}, + task_io::compression::{ClpSCompressionOption, CompressionTaskOutput, S3InputSource}, }; use spider_client::{SpiderClient, error::ClientError}; use spider_core::{ @@ -86,7 +86,7 @@ impl S3CompressionJobSubmitter for SpiderClient { )?), ], outputs: vec![DataTypeDescriptor::Value( - ValueTypeDescriptor::struct_from_name("CompressionTaskOutput")?, + ValueTypeDescriptor::struct_from_name(stringify!(CompressionTaskOutput))?, )], input_sources: None, })?; From a7a96d4ff41d3e9721aea5894f4798c360aa5955 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 21 Jul 2026 17:08:29 -0400 Subject: [PATCH 15/16] Stringify fix --- .../src/compression_job_submitter/spider.rs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/components/compression-coordinator/src/compression_job_submitter/spider.rs b/components/compression-coordinator/src/compression_job_submitter/spider.rs index fa33c97473..9b83353838 100644 --- a/components/compression-coordinator/src/compression_job_submitter/spider.rs +++ b/components/compression-coordinator/src/compression_job_submitter/spider.rs @@ -75,15 +75,15 @@ impl S3CompressionJobSubmitter for SpiderClient { }, execution_policy: Some(execution_policy), inputs: vec![ - DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - stringify!(ClpSCompressionOption), - )?), - DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - stringify!(Option), - )?), - DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name( - stringify!(S3InputSource), - )?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name(stringify!( + ClpSCompressionOption + ))?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name(stringify!( + Option + ))?), + DataTypeDescriptor::Value(ValueTypeDescriptor::struct_from_name(stringify!( + S3InputSource + ))?), ], outputs: vec![DataTypeDescriptor::Value( ValueTypeDescriptor::struct_from_name(stringify!(CompressionTaskOutput))?, @@ -135,7 +135,9 @@ impl S3CompressionJobSubmitter for SpiderClient { break state; } tokio::time::sleep(backoff).await; - backoff = backoff.saturating_mul(POLL_BACKOFF_FACTOR).min(max_poll_backoff); + backoff = backoff + .saturating_mul(POLL_BACKOFF_FACTOR) + .min(max_poll_backoff); }; Ok(match terminal_state { From f1aaf66f1c54c4007408abfbb7d102db3aae61fa Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 21 Jul 2026 21:22:15 -0400 Subject: [PATCH 16/16] Add key prefix mismatch error --- components/compression-coordinator/src/error.rs | 3 +++ components/compression-coordinator/src/partition.rs | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/components/compression-coordinator/src/error.rs b/components/compression-coordinator/src/error.rs index 67a32afbdd..68fb899cc9 100644 --- a/components/compression-coordinator/src/error.rs +++ b/components/compression-coordinator/src/error.rs @@ -6,6 +6,9 @@ pub enum Error { #[error("S3 bucket mismatch: expected `{0}`, but got `{1}`")] S3BucketMismatch(String, String), + #[error("S3 key prefix mismatch: expected key to start with `{0}`, but got `{1}`")] + S3KeyPrefixMismatch(String, String), + /// Failed to build or serialize the compression task graph. #[error("failed to build the compression task graph: {0}")] TaskGraph(#[from] spider_core::task::Error), diff --git a/components/compression-coordinator/src/partition.rs b/components/compression-coordinator/src/partition.rs index 907ab9fc90..75f85707f1 100644 --- a/components/compression-coordinator/src/partition.rs +++ b/components/compression-coordinator/src/partition.rs @@ -56,6 +56,8 @@ impl CompressionInputBuilder { /// /// * [`Error::S3BucketMismatch`] if `metadata`'s bucket differs from the buffer's configured /// bucket. + /// * [`Error::S3KeyPrefixMismatch`] if `metadata`'s key doesn't start with the buffer's + /// configured key prefix. pub fn add(&mut self, metadata: ObjectMetadata) -> Result<(), Error> { if metadata.bucket != self.s3_config.bucket { return Err(Error::S3BucketMismatch( @@ -64,6 +66,17 @@ impl CompressionInputBuilder { )); } + if !metadata + .key + .as_str() + .starts_with(self.s3_config.key_prefix.as_str()) + { + return Err(Error::S3KeyPrefixMismatch( + String::from(self.s3_config.key_prefix.clone()), + String::from(metadata.key), + )); + } + let file = FileMetadata::new(String::from(metadata.key), metadata.size); self.total_buffered_size += file.estimated_size; self.buffer.push(file);