From 0a6f0da7c8c7d0b05bb7d3df16ca8ca998a05c25 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Tue, 16 Jun 2026 12:51:36 +0200 Subject: [PATCH 1/4] test(parquet): green RFC0013.1 + .7 via a testcontainers + LocalStack CI lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stand up the S3 integration lane the RFC 0013 §6 testing strategy specified: - RFC0013.1 (S3 round-trip == local): encode a 300-row batch, put/get it through `Store::s3` (LocalStack) and the local backend, and assert the bytes match and decode back to the same records. - RFC0013.7 (endpoint override): a put/get/delete through a store configured purely via `S3Config::endpoint` behaves like AWS S3. Both spin LocalStack (S3-compatible, Apache-2.0) via testcontainers and create the bucket with the image's own `awslocal` (no S3-client dep). They are `#[ignore]`d — testcontainers needs a Docker-API runtime, which GitHub Actions has but local containerd does not — and a new CI job `s3-integration` (required, in `ci-success`'s `needs`) runs them by name via `--ignored`, with `AWS_*` env feeding object_store's credential chain. object_store's default path-style addressing already suits LocalStack, so `Store::s3` is unchanged. Supporting: - Add the `S3Config::new` + `with_endpoint`/`with_region`/`with_prefix` builder — `S3Config` is `#[non_exhaustive]`, so external crates (this test, and the server later) could not construct one before. Internal test updated to use it. - Bump dev-only `time` 0.3.45 → 0.3.47 (RUSTSEC-2026-0009, RFC 2822 DoS) pulled transitively by testcontainers; cargo-deny green. It's a dev dep, so the MSRV 1.85 library claim is unaffected. These S3 ITs are CI-verified-only (can't run on local containerd); validated via the `s3-integration` Actions job. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 28 +- Cargo.lock | 577 +++++++++++++++++- crates/ourios-parquet/Cargo.toml | 6 + crates/ourios-parquet/src/store.rs | 47 +- .../tests/rfc0013_object_store.rs | 103 +++- 5 files changed, 726 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75449386c..fa35f0113 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,32 @@ jobs: - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - run: cargo test --all-features + # RFC 0013 S3 integration tests (RFC0013.1/.7, later .3/.4): spin LocalStack + # (S3-compatible, Apache-2.0) via testcontainers and drive `Store::s3` at it + # through the endpoint override. The tests are `#[ignore]`d (they need a + # Docker-API runtime, which GitHub Actions has but local containerd does + # not), so this job runs them explicitly by name via `--ignored`. The + # `AWS_*` env feeds object_store's credential chain — LocalStack accepts any + # credentials. A required check (in `ci-success`'s `needs`). + s3-integration: + name: s3 integration (localstack) + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (pinned); channel via toolchain input + with: + toolchain: stable + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - run: > + cargo test -p ourios-parquet --test rfc0013_object_store -- + --ignored --exact + rfc0013_1_round_trip_through_s3_backend + rfc0013_7_s3_compatible_endpoint_via_override + book: name: mdbook build runs-on: ubuntu-latest @@ -246,7 +272,7 @@ jobs: ci-success: name: CI Success runs-on: ubuntu-latest - needs: [fmt, clippy, docs, test, book, semconv, deny] + needs: [fmt, clippy, docs, test, book, semconv, deny, s3-integration] if: always() steps: - name: Verify required checks diff --git a/Cargo.lock b/Cargo.lock index 390de54da..3f46caac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -353,7 +353,7 @@ dependencies = [ "arrow-select 58.3.0", "chrono", "half", - "indexmap", + "indexmap 2.14.0", "itoa", "lexical-core", "memchr", @@ -451,6 +451,44 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "astral-tokio-tar" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb50a7aae84a03bf55b067832bc376f4961b790c97e64d3eacee97d389b90277" +dependencies = [ + "filetime", + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -579,6 +617,80 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" +dependencies = [ + "async-stream", + "base64", + "bitflags", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.9.4", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.52.1-rc.29.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" +dependencies = [ + "base64", + "bollard-buildkit-proto", + "bytes", + "prost", + "serde", + "serde_json", + "serde_repr", + "time", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -964,6 +1076,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -1092,7 +1239,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", @@ -1288,7 +1435,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "paste", "serde_json", @@ -1302,7 +1449,7 @@ checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", "datafusion-common", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "paste", ] @@ -1437,7 +1584,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "log", "regex", @@ -1459,7 +1606,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", "paste", @@ -1494,7 +1641,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", ] @@ -1540,7 +1687,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.14.0", "log", "num-traits", @@ -1580,6 +1727,16 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + [[package]] name = "digest" version = "0.10.7" @@ -1601,6 +1758,23 @@ dependencies = [ "syn", ] +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64", + "serde", + "serde_json", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -1632,6 +1806,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "fancy-regex" version = "0.18.0" @@ -1649,6 +1833,27 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "portable-atomic", + "rand 0.10.1", + "web-time", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1887,7 +2092,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -1906,6 +2111,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1956,6 +2167,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "1.4.1" @@ -2029,6 +2249,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.27.9" @@ -2081,6 +2316,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2192,6 +2442,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -2213,6 +2469,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -2544,6 +2811,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -2809,6 +3082,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "testcontainers-modules", "tokio", "uuid", ] @@ -2969,6 +3243,31 @@ dependencies = [ "zstd", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn", +] + [[package]] name = "paste" version = "1.0.15" @@ -2989,7 +3288,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "serde", ] @@ -3086,6 +3385,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -3150,7 +3455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -3497,6 +3802,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -3580,6 +3886,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3664,6 +3994,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3676,13 +4017,44 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -3774,6 +4146,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3824,6 +4219,46 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "testcontainers" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" +dependencies = [ + "astral-tokio-tar", + "async-trait", + "bollard", + "bytes", + "docker_credential", + "either", + "etcetera", + "ferroid", + "futures", + "http", + "itertools 0.14.0", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5985fde5befe4ffa77a052e035e16c2da86e8bae301baa9f9904ad3c494d357" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -3855,6 +4290,37 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -4020,7 +4486,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -4158,6 +4624,33 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -4168,8 +4661,15 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4339,7 +4839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -4365,7 +4865,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "semver", ] @@ -4389,6 +4889,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -4398,6 +4914,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -4466,6 +4988,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -4647,7 +5178,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap", + "indexmap 2.14.0", "prettyplease", "syn", "wasm-metadata", @@ -4678,7 +5209,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -4697,7 +5228,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.14.0", "log", "semver", "serde", @@ -4713,6 +5244,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/crates/ourios-parquet/Cargo.toml b/crates/ourios-parquet/Cargo.toml index 8807a94cc..c6fdb9fab 100644 --- a/crates/ourios-parquet/Cargo.toml +++ b/crates/ourios-parquet/Cargo.toml @@ -75,6 +75,12 @@ proptest = "1" # Async runtime for the RFC 0013 `Store` round-trip test (the I/O surface # is async via `object_store`). Already in the workspace tree. tokio = { version = "1", default-features = false, features = ["rt", "macros"] } +# RFC 0013 S3 integration tests: spin LocalStack (S3-compatible, Apache-2.0) +# via testcontainers and point `Store::s3` at it through the endpoint override. +# The S3 tests are `#[ignore]`d (run only by the `s3-integration` CI job via +# `--ignored` + a name filter, where a Docker-API runtime is available); the +# default `cargo test` compiles but skips them. +testcontainers-modules = { version = "0.15", features = ["localstack"] } [lints] workspace = true diff --git a/crates/ourios-parquet/src/store.rs b/crates/ourios-parquet/src/store.rs index fda2e150f..01bf01a95 100644 --- a/crates/ourios-parquet/src/store.rs +++ b/crates/ourios-parquet/src/store.rs @@ -132,6 +132,43 @@ pub struct S3Config { pub prefix: Option, } +impl S3Config { + /// Config for `bucket` (required); endpoint, region, and prefix start + /// unset — add them with the `with_*` builders. `S3Config` is + /// `#[non_exhaustive]`, so this is the constructor external callers use. + #[must_use] + pub fn new(bucket: impl Into) -> Self { + Self { + bucket: bucket.into(), + endpoint: None, + region: None, + prefix: None, + } + } + + /// Set the endpoint override for an S3-compatible store (Hetzner, R2, + /// `LocalStack`, …). + #[must_use] + pub fn with_endpoint(mut self, endpoint: impl Into) -> Self { + self.endpoint = Some(endpoint.into()); + self + } + + /// Set the region. + #[must_use] + pub fn with_region(mut self, region: impl Into) -> Self { + self.region = Some(region.into()); + self + } + + /// Set the key prefix (the store root within the bucket). + #[must_use] + pub fn with_prefix(mut self, prefix: impl Into) -> Self { + self.prefix = Some(prefix.into()); + self + } +} + /// Errors from constructing or addressing a [`Store`]. #[derive(Debug)] #[non_exhaustive] @@ -383,12 +420,10 @@ mod tests { /// endpoint. #[test] fn s3_constructs_from_a_valid_config() { - let cfg = S3Config { - bucket: "ourios-test".to_string(), - endpoint: Some("https://s3.example.invalid".to_string()), - region: Some("eu-central-1".to_string()), - prefix: Some("ourios".to_string()), - }; + let cfg = S3Config::new("ourios-test") + .with_endpoint("https://s3.example.invalid") + .with_region("eu-central-1") + .with_prefix("ourios"); let store = Store::s3(cfg).expect("s3 construct"); assert_eq!(store.prefix().as_ref(), "ourios", "prefix is honoured"); } diff --git a/crates/ourios-parquet/tests/rfc0013_object_store.rs b/crates/ourios-parquet/tests/rfc0013_object_store.rs index 90ea8e595..f9c46bbf7 100644 --- a/crates/ourios-parquet/tests/rfc0013_object_store.rs +++ b/crates/ourios-parquet/tests/rfc0013_object_store.rs @@ -15,7 +15,55 @@ use ourios_core::audit::ParamType; use ourios_core::record::{BodyKind, MinedRecord, Param}; use ourios_core::tenant::TenantId; -use ourios_parquet::{PartitionKey, Reader, S3Config, Store, Writer}; +use ourios_parquet::{ + DEFAULT_ZSTD_LEVEL, PartitionKey, Reader, S3Config, Store, Writer, encode_records_to_parquet, +}; +use testcontainers_modules::localstack::LocalStack; +use testcontainers_modules::testcontainers::core::ExecCommand; +use testcontainers_modules::testcontainers::runners::AsyncRunner; +use testcontainers_modules::testcontainers::{ContainerAsync, ImageExt}; + +/// Start a `LocalStack` S3 container, create `bucket` with the image's own +/// `awslocal` (no S3-client dependency), and return the running container +/// (the caller keeps it alive — dropping it stops `LocalStack`) paired with a +/// `Store::s3` pointed at it via the endpoint override. Credentials come from +/// the `AWS_*` env the `s3-integration` CI job sets. +async fn localstack_s3(bucket: &str) -> (ContainerAsync, Store) { + let container = LocalStack::default() + .with_env_var("SERVICES", "s3") + .start() + .await + .expect("start localstack"); + let host = container.get_host().await.expect("container host"); + let port = container + .get_host_port_ipv4(4566) + .await + .expect("container port"); + let endpoint = format!("http://{host}:{port}"); + + let mb = container + .exec(ExecCommand::new([ + "awslocal".to_string(), + "s3".to_string(), + "mb".to_string(), + format!("s3://{bucket}"), + ])) + .await + .expect("exec awslocal s3 mb"); + assert_eq!( + mb.exit_code().await.expect("mb exit code"), + Some(0), + "awslocal s3 mb must succeed", + ); + + let store = Store::s3( + S3Config::new(bucket) + .with_endpoint(endpoint) + .with_region("us-east-1"), + ) + .expect("build s3 store"); + (container, store) +} /// A clean-round-trip record for `tenant` at a fixed in-hour offset `i`. fn rec_for(tenant: &str, i: u64) -> MinedRecord { @@ -63,11 +111,37 @@ fn write_through_store( /// Scenario RFC0013.1 — a `MinedRecord` batch written and read through the `AmazonS3` /// backend recovers byte-for-byte against the local backend. /// See `docs/rfcs/0013-object-storage.md` §5. -#[test] -#[ignore = "RFC0013.1 — red until the S3 backend + writer/reader migration land"] -fn rfc0013_1_round_trip_through_s3_backend() { - let _ = Store::s3; - todo!("RFC0013.1: S3 round-trip == local round-trip (MinIO testcontainer)") +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "RFC0013.1 — S3 integration; run via the `s3-integration` CI job (needs Docker + AWS_* env)"] +async fn rfc0013_1_round_trip_through_s3_backend() { + let records: Vec = (0..300).map(|i| rec_for("tenant-a", i)).collect(); + let bytes = encode_records_to_parquet(&records, DEFAULT_ZSTD_LEVEL).expect("encode"); + let key = "data/tenant_id=tenant-a/year=2026/month=04/day=02/hour=10/file.parquet"; + + let (_node, s3) = localstack_s3("ourios-it-roundtrip").await; + s3.put(key, bytes.clone()).await.expect("s3 put"); + let s3_bytes = s3.get(key).await.expect("s3 get"); + + // The same bytes through the local backend. + let dir = tempfile::TempDir::new().expect("temp dir"); + let local = Store::local(dir.path()).expect("local store"); + local.put(key, bytes.clone()).await.expect("local put"); + let local_bytes = local.get(key).await.expect("local get"); + + assert_eq!(s3_bytes, bytes, "S3 returns the written bytes"); + assert_eq!( + s3_bytes, local_bytes, + "S3 and local recover identical bytes" + ); + + let decoded = Reader::open_bytes(bytes::Bytes::from(s3_bytes)) + .expect("open_bytes") + .read_all() + .expect("read_all"); + assert_eq!( + decoded, records, + "records recover byte-for-byte through the S3 backend" + ); } /// Scenario RFC0013.2 — the existing RFC 0005 / 0009 suites pass unchanged against the @@ -179,10 +253,19 @@ fn rfc0013_6_wal_stays_local() { /// Scenario RFC0013.7 — an S3-compatible store (`MinIO`) configured via an endpoint /// override (RFC 0004) reads and writes exactly as AWS S3. /// See `docs/rfcs/0013-object-storage.md` §5. -#[test] -#[ignore = "RFC0013.7 — red until the S3 backend honours endpoint overrides"] -fn rfc0013_7_s3_compatible_endpoint_via_override() { - todo!("RFC0013.7: MinIO endpoint override works like AWS S3") +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "RFC0013.7 — S3 integration; run via the `s3-integration` CI job"] +async fn rfc0013_7_s3_compatible_endpoint_via_override() { + // The store is configured purely through `S3Config::endpoint` (the RFC 0004 + // override) pointing at LocalStack; put/get/delete behave like AWS S3. + let (_node, s3) = localstack_s3("ourios-it-endpoint").await; + let key = "data/tenant_id=t/probe.bin"; + s3.put(key, b"endpoint-override".to_vec()) + .await + .expect("put"); + assert_eq!(s3.get(key).await.expect("get"), b"endpoint-override"); + s3.delete(key).await.expect("delete"); + assert!(s3.get(key).await.is_err(), "object gone after delete"); } /// Scenario RFC0013.8 — the RFC 0005 §3.9 reader forward-compat contract (absent From 88a42d457631e0e4d4f2f038684c1d6f80aeb997 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Tue, 16 Jun 2026 12:58:23 +0200 Subject: [PATCH 2/4] test(parquet): drain exec output before exit_code in the localstack bucket-create --- crates/ourios-parquet/tests/rfc0013_object_store.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/ourios-parquet/tests/rfc0013_object_store.rs b/crates/ourios-parquet/tests/rfc0013_object_store.rs index f9c46bbf7..8df80d232 100644 --- a/crates/ourios-parquet/tests/rfc0013_object_store.rs +++ b/crates/ourios-parquet/tests/rfc0013_object_store.rs @@ -41,7 +41,7 @@ async fn localstack_s3(bucket: &str) -> (ContainerAsync, Store) { .expect("container port"); let endpoint = format!("http://{host}:{port}"); - let mb = container + let mut mb = container .exec(ExecCommand::new([ "awslocal".to_string(), "s3".to_string(), @@ -50,10 +50,17 @@ async fn localstack_s3(bucket: &str) -> (ContainerAsync, Store) { ])) .await .expect("exec awslocal s3 mb"); + // Drain both streams before reading the exit code — testcontainers reports + // `exit_code()` as `None` until the exec's output has been consumed. + let stdout = + String::from_utf8_lossy(&mb.stdout_to_vec().await.expect("mb stdout")).into_owned(); + let stderr = + String::from_utf8_lossy(&mb.stderr_to_vec().await.expect("mb stderr")).into_owned(); + let code = mb.exit_code().await.expect("mb exit code"); assert_eq!( - mb.exit_code().await.expect("mb exit code"), + code, Some(0), - "awslocal s3 mb must succeed", + "awslocal s3 mb failed (code {code:?}): stdout={stdout:?} stderr={stderr:?}", ); let store = Store::s3( From 8db27394eaaa3a76f527c5fed56bac2e2fe7e75c Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Tue, 16 Jun 2026 13:36:36 +0200 Subject: [PATCH 3/4] test(parquet): self-contained tokio dev-dep + assert NotFound post-delete (copilot) --- crates/ourios-parquet/Cargo.toml | 8 +++++--- crates/ourios-parquet/tests/rfc0013_object_store.rs | 6 +++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/ourios-parquet/Cargo.toml b/crates/ourios-parquet/Cargo.toml index c6fdb9fab..82f655669 100644 --- a/crates/ourios-parquet/Cargo.toml +++ b/crates/ourios-parquet/Cargo.toml @@ -72,9 +72,11 @@ tempfile = "3" # Property tests for the compaction row-conservation invariant # (RFC0009.2 / CLAUDE.md §6.2 — invariants get a property test). proptest = "1" -# Async runtime for the RFC 0013 `Store` round-trip test (the I/O surface -# is async via `object_store`). Already in the workspace tree. -tokio = { version = "1", default-features = false, features = ["rt", "macros"] } +# Async runtime for the RFC 0013 `Store` round-trip tests (the I/O surface is +# async via `object_store`). `rt-multi-thread` so the S3 integration tests' +# `#[tokio::test(flavor = "multi_thread")]` is self-contained rather than +# relying on the main dependency's feature set. Already in the workspace tree. +tokio = { version = "1", default-features = false, features = ["rt", "rt-multi-thread", "macros"] } # RFC 0013 S3 integration tests: spin LocalStack (S3-compatible, Apache-2.0) # via testcontainers and point `Store::s3` at it through the endpoint override. # The S3 tests are `#[ignore]`d (run only by the `s3-integration` CI job via diff --git a/crates/ourios-parquet/tests/rfc0013_object_store.rs b/crates/ourios-parquet/tests/rfc0013_object_store.rs index 8df80d232..7891ac290 100644 --- a/crates/ourios-parquet/tests/rfc0013_object_store.rs +++ b/crates/ourios-parquet/tests/rfc0013_object_store.rs @@ -272,7 +272,11 @@ async fn rfc0013_7_s3_compatible_endpoint_via_override() { .expect("put"); assert_eq!(s3.get(key).await.expect("get"), b"endpoint-override"); s3.delete(key).await.expect("delete"); - assert!(s3.get(key).await.is_err(), "object gone after delete"); + let err = s3.get(key).await.expect_err("object gone after delete"); + assert!( + err.is_not_found(), + "post-delete get should be NotFound, got {err:?}", + ); } /// Scenario RFC0013.8 — the RFC 0005 §3.9 reader forward-compat contract (absent From 57ae470849967d384fdea26316d6d3872f849c96 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Tue, 16 Jun 2026 13:46:36 +0200 Subject: [PATCH 4/4] docs(parquet): clarify S3Config::new is the preferred (not only) constructor (copilot) --- crates/ourios-parquet/src/store.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/ourios-parquet/src/store.rs b/crates/ourios-parquet/src/store.rs index 01bf01a95..9be8ebc57 100644 --- a/crates/ourios-parquet/src/store.rs +++ b/crates/ourios-parquet/src/store.rs @@ -134,8 +134,10 @@ pub struct S3Config { impl S3Config { /// Config for `bucket` (required); endpoint, region, and prefix start - /// unset — add them with the `with_*` builders. `S3Config` is - /// `#[non_exhaustive]`, so this is the constructor external callers use. + /// unset — add them with the `with_*` builders. The preferred way to build + /// an `S3Config` (it's `#[non_exhaustive]`, so external callers can't use a + /// struct literal; `S3Config::default()` plus setting the public fields + /// also works, but `bucket` then defaults to the invalid empty string). #[must_use] pub fn new(bucket: impl Into) -> Self { Self {