diff --git a/Cargo.lock b/Cargo.lock index 516979447..0ef6582a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -16,6 +22,36 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -93,6 +129,41 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +dependencies = [ + "arrow-arith", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-csv", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "55.2.0" @@ -100,15 +171,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", "chrono", "half", "hashbrown 0.15.5", "num", ] +[[package]] +name = "arrow-array" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +dependencies = [ + "ahash", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "55.2.0" @@ -120,17 +210,29 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", "atoi", "base64", "chrono", @@ -140,29 +242,145 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ord", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +dependencies = [ + "arrow-array 58.3.0", + "arrow-cast 58.3.0", + "arrow-schema 58.3.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 55.2.0", + "arrow-schema 55.2.0", "half", "num", ] +[[package]] +name = "arrow-data" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +dependencies = [ + "arrow-buffer 58.3.0", + "arrow-schema 58.3.0", + "half", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-ipc" version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", + "flatbuffers", +] + +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", "flatbuffers", + "lz4_flex", +] + +[[package]] +name = "arrow-json" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-ord", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", +] + +[[package]] +name = "arrow-row" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "half", ] [[package]] @@ -171,6 +389,16 @@ version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +[[package]] +name = "arrow-schema" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +dependencies = [ + "serde_core", + "serde_json", +] + [[package]] name = "arrow-select" version = "55.2.0" @@ -178,13 +406,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-data 55.2.0", + "arrow-schema 55.2.0", "num", ] +[[package]] +name = "arrow-select" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +dependencies = [ + "ahash", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atoi" version = "2.0.0" @@ -235,6 +505,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "8.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -282,6 +573,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + [[package]] name = "clap" version = "4.6.1" @@ -326,93 +627,740 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-hex" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e2a781ebdf4467d1428dc4593067825fb646f6871475098d8577421af73558" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +dependencies = [ + "arrow", + "arrow-schema 58.3.0", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet 58.3.0", + "rand", + "regex", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools", + "log", + "object_store", +] + +[[package]] +name = "datafusion-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc 58.3.0", + "chrono", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "libc", + "log", + "object_store", + "parquet 58.3.0", + "paste", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "glob", + "itertools", + "log", + "object_store", + "rand", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +dependencies = [ + "arrow", + "arrow-ipc 58.3.0", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet 58.3.0", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" + +[[package]] +name = "datafusion-execution" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +dependencies = [ + "arrow", + "arrow-buffer 58.3.0", + "async-trait", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "itertools", + "paste", + "serde_json", +] + +[[package]] +name = "datafusion-expr-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +dependencies = [ + "arrow", + "arrow-buffer 58.3.0", + "base64", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hex", + "itertools", + "log", + "memchr", + "num-traits", + "rand", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "num-traits", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-table" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] [[package]] -name = "const-hex" -version = "1.19.1" +name = "datafusion-functions-window-common" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e2a781ebdf4467d1428dc4593067825fb646f6871475098d8577421af73558" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "proptest", - "serde_core", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] -name = "const-random" -version = "0.1.18" +name = "datafusion-macros" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ - "const-random-macro", + "datafusion-doc", + "quote", + "syn", ] [[package]] -name = "const-random-macro" -version = "0.1.16" +name = "datafusion-optimizer" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ - "getrandom 0.2.17", - "once_cell", - "tiny-keccak", + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap", + "itertools", + "log", + "regex", + "regex-syntax", ] [[package]] -name = "constant_time_eq" -version = "0.4.2" +name = "datafusion-physical-expr" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "parking_lot", + "paste", + "petgraph", + "tokio", +] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "datafusion-physical-expr-adapter" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools", +] [[package]] -name = "cpufeatures" -version = "0.2.17" +name = "datafusion-physical-expr-common" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ - "libc", + "ahash", + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "parking_lot", ] [[package]] -name = "cpufeatures" -version = "0.3.0" +name = "datafusion-physical-optimizer" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ - "libc", + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools", ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "datafusion-physical-plan" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ - "rustc_version", + "ahash", + "arrow", + "arrow-ord", + "arrow-schema 58.3.0", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", ] [[package]] -name = "crunchy" -version = "0.2.4" +name = "datafusion-pruning" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools", + "log", +] [[package]] -name = "crypto-common" -version = "0.1.7" +name = "datafusion-session" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ - "generic-array", - "typenum", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", ] [[package]] @@ -425,6 +1373,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -459,6 +1418,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "25.12.19" @@ -469,12 +1434,52 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -482,6 +1487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -501,6 +1507,12 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + [[package]] name = "futures-macro" version = "0.3.32" @@ -530,10 +1542,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -584,6 +1599,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "half" version = "2.7.1" @@ -596,13 +1617,30 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -617,6 +1655,28 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -641,12 +1701,114 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -789,18 +1951,52 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num" version = "0.4.3" @@ -875,6 +2071,32 @@ dependencies = [ "libm", ] +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools", + "parking_lot", + "percent-encoding", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -969,11 +2191,11 @@ dependencies = [ name = "ourios-parquet" version = "0.0.0" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 55.2.0", + "arrow-schema 55.2.0", "chrono", "ourios-core", - "parquet", + "parquet 55.2.0", "serde", "serde_json", "tempfile", @@ -984,7 +2206,11 @@ dependencies = [ name = "ourios-querier" version = "0.0.0" dependencies = [ + "datafusion", "ourios-core", + "ourios-parquet", + "tempfile", + "tokio", ] [[package]] @@ -997,30 +2223,89 @@ dependencies = [ "uuid", ] +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +dependencies = [ + "ahash", + "arrow-array 55.2.0", + "arrow-buffer 55.2.0", + "arrow-cast 55.2.0", + "arrow-data 55.2.0", + "arrow-ipc 55.2.0", + "arrow-schema 55.2.0", + "arrow-select 55.2.0", + "base64", + "bytes", + "chrono", + "half", + "hashbrown 0.15.5", + "num", + "num-bigint", + "paste", + "seq-macro", + "thrift", + "twox-hash", + "zstd", +] + [[package]] name = "parquet" -version = "55.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-data 58.3.0", + "arrow-ipc 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", "base64", + "brotli", "bytes", "chrono", + "flate2", + "futures", "half", - "hashbrown 0.15.5", - "num", + "hashbrown 0.17.1", + "lz4_flex", "num-bigint", + "num-integer", + "num-traits", + "object_store", "paste", "seq-macro", + "simdutf8", + "snap", "thrift", + "tokio", "twox-hash", "zstd", ] @@ -1031,6 +2316,42 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1049,6 +2370,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1174,6 +2504,38 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.10" @@ -1214,6 +2576,21 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "1.0.28" @@ -1286,12 +2663,48 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" @@ -1309,6 +2722,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1362,6 +2786,94 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "twox-hash" version = "2.1.2" @@ -1386,12 +2898,42 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1415,6 +2957,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1452,6 +3004,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.121" @@ -1518,6 +3080,25 @@ dependencies = [ "semver", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1680,6 +3261,35 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.48" @@ -1700,6 +3310,66 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + [[package]] name = "zmij" version = "1.0.21" diff --git a/crates/ourios-querier/Cargo.toml b/crates/ourios-querier/Cargo.toml index d0905594e..f4f937e25 100644 --- a/crates/ourios-querier/Cargo.toml +++ b/crates/ourios-querier/Cargo.toml @@ -15,12 +15,23 @@ path = "src/lib.rs" [dependencies] # `TenantId` for the per-request tenant scope (CLAUDE.md §3.7). ourios-core = { path = "../ourios-core" } -# NOTE: `ourios-parquet` (the RFC 0005 reader this queries) and -# `datafusion` (pillar #3 — the execution engine, kept a -# `pub(crate)` dep so its types never cross the public API per -# hazard §4.6) land with the execution slice. The red-gate -# scaffold doesn't reference them, and `datafusion` is heavy, so -# they're omitted here to keep the gate build minimal. +# The RFC 0005 store this queries — used here for the canonical +# `percent_encode_tenant` so the tenant partition-dir path matches +# the writer's byte-for-byte (single source of truth, no drift). +ourios-parquet = { path = "../ourios-parquet" } +# Pillar #3 — the execution engine. A plain dep, but its types are +# kept strictly internal: no `datafusion`/`arrow` type appears in +# any public signature or error (hazard §4.6). Pulls arrow/parquet +# 58 vs ourios-parquet's 55; harmless because the two never cross +# in memory (DataFusion reads Parquet files from disk). +datafusion = { version = "53", default-features = false, features = ["parquet"] } + +[dev-dependencies] +# Async runtime for the execution integration tests; the writer +# (ourios-parquet, a normal dep) lays down fixture Parquet, and +# tempfile gives each test a scratch bucket. +tokio = { version = "1", features = ["rt", "macros"] } +tempfile = "3" [lints] workspace = true diff --git a/crates/ourios-querier/src/lib.rs b/crates/ourios-querier/src/lib.rs index c1fdd5c57..407563360 100644 --- a/crates/ourios-querier/src/lib.rs +++ b/crates/ourios-querier/src/lib.rs @@ -1,45 +1,66 @@ //! `ourios-querier` — RFC 0007 querier (pillar #3, `DataFusion`). //! -//! **Status: red gate (scaffold).** The public API surface from -//! RFC 0007 §4.1 is in place but [`Querier::run`] returns -//! `unimplemented!()`. The `#[ignore]`'d acceptance tests under -//! `tests/` enumerate the RFC 0007 §5 scenarios (RFC0007.1 -//! through .5) the implementation must satisfy before the RFC -//! moves `specified → red → green`. +//! **Status: execution slice 1.** [`Querier::run`] executes a +//! minimal query — tenant scope + optional time range + optional +//! template-exact id — against the RFC 0005 Parquet store via +//! `DataFusion`, returning a matching-row count. Tenant isolation +//! (RFC0007.5) is live + tested; the row-group pruning stats that +//! prove B1 (RFC0007.1) and the B2 latency bench come next. //! -//! This crate is the **read path**: it lowers a logs-DSL query -//! (RFC 0002) to a `DataFusion` `LogicalPlan`, executes it against -//! the RFC 0005 Parquet store with predicate pushdown -//! (partition pruning + row-group skipping on `template_id` / -//! `time_unix_nano` / severity, RFC 0005 §3.3/§3.6), and returns -//! typed results — **without** leaking `DataFusion` or SQL through -//! the public API (hazard `CLAUDE.md` §4.6). It depends on the -//! shipped RFC 0005 reader, not on the WAL or receiver. +//! This crate is the **read path**: it runs the query against the +//! RFC 0005 store — scoped to the tenant's partition directory, +//! with `template_id` / `time_unix_nano` column filters (RFC 0005 +//! §3.3/§3.6) — and returns results **without** leaking +//! `DataFusion` or SQL through the public API (hazard `CLAUDE.md` +//! §4.6). It reads the shipped RFC 0005 store; it needs neither +//! the WAL nor the receiver. //! -//! **Deferred (execution slice):** the DSL→plan lowering and -//! query execution need RFC 0002's still-undecided Branch A/B -//! syntax. RFC 0007 scopes *this* layer — surface, pushdown -//! contract, B1/B2 criteria — as branch-independent, so the -//! scaffold + criteria land now; execution follows RFC 0002. +//! (Partition-level *time* pruning — deriving `year/month/day/hour` +//! path bounds from the time range so whole directories are +//! skipped — is a later refinement; today the time bound is a +//! column predicate, and the row-group skipping it enables is what +//! slice 2 / B1 measures.) +//! +//! **Throwaway query surface.** [`QueryRequest`] is intentionally +//! minimal — just the predicates B1/B2 need. The real logs DSL +//! (RFC 0002) is deferred until B1/B2 prove the query thesis is +//! worth a stable language; until then this surface may change +//! freely (maintainer decision). #![deny(unsafe_code)] +use std::path::PathBuf; +use std::sync::Arc; + +use datafusion::arrow::datatypes::DataType; +use datafusion::common::ScalarValue; +use datafusion::datasource::file_format::parquet::ParquetFormat; +use datafusion::datasource::listing::{ + ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, +}; +use datafusion::error::DataFusionError; +use datafusion::prelude::{SessionContext, col, lit}; use ourios_core::tenant::TenantId; +use ourios_parquet::columns; +use ourios_parquet::percent_encode_tenant; -/// A logs-DSL query to execute. Per RFC 0007 §3.1 the querier -/// consumes the AST that RFC 0002 produces; that parsed-query -/// field is deferred until RFC 0002's Branch A/B decision lands, -/// so the request currently carries only the tenant scope and -/// optional time bounds (the partition-prune keys). +/// A logs query to execute. **Throwaway surface** while the query +/// thesis (B1/B2) is unproven — per the maintainer decision, DSL +/// contracts (RFC 0002) are deferred until B1/B2 say the querier +/// is worth a stable language. So this carries only the minimal +/// predicates B1/B2 need: tenant scope, optional time bounds, and +/// optional template-exact id — exactly the RFC 0005 §3.3 +/// pushdown keys. #[derive(Debug, Clone)] pub struct QueryRequest { - /// Tenant whose data the query is scoped to. A query without - /// a tenant is a usage error, not a cross-tenant scan - /// (`CLAUDE.md` §3.7; RFC0007.5). + /// Tenant whose data the query is scoped to. Enforced + /// structurally — the querier only ever reads under this + /// tenant's partition directory (`CLAUDE.md` §3.7; RFC0007.5). pub tenant: TenantId, - /// Optional `[start, end)` `time_unix_nano` bounds — a - /// partition-prune key (RFC 0005 §3.3 pushdown set). + /// Optional `[start, end)` `time_unix_nano` bounds. pub time_range: Option<(u64, u64)>, + /// Optional template-exact filter (B2 — `template_id` equality). + pub template_id: Option, } /// Pruning / IO accounting for one query, surfaced so B1 @@ -65,6 +86,11 @@ pub struct QueryStats { /// the B1/B2 gates assert on. #[derive(Debug, Clone, Default)] pub struct QueryResult { + /// Number of matching rows. (The typed-row payload — projected + /// columns — lands when there's a query thesis worth shaping it + /// around; B1/B2 only need the count + stats. Stays free of + /// arrow `RecordBatch` leakage per §4.6.) + pub rows: u64, pub stats: QueryStats, } @@ -85,6 +111,11 @@ pub enum QueryError { /// The query failed to compile from the logs DSL (RFC 0002). InvalidQuery { detail: String }, /// Object-storage / Parquet read failure during execution. + /// `detail` carries the underlying engine message for + /// `Debug`/logs **only** — it is deliberately *not* rendered + /// by `Display`, because `DataFusion`/arrow error text leaks + /// implementation specifics the public surface must not expose + /// (hazard §4.6 / RFC0007.3). Storage { detail: String }, } @@ -93,43 +124,188 @@ impl std::fmt::Display for QueryError { match self { Self::TenantRequired => write!(f, "query has no tenant scope"), Self::InvalidQuery { detail } => write!(f, "invalid query: {detail}"), - Self::Storage { detail } => write!(f, "storage read failed: {detail}"), + // No `detail` here on purpose: the underlying engine + // message would leak `DataFusion`/SQL specifics (§4.6). + // The detail is preserved on the variant for `Debug`. + Self::Storage { .. } => write!(f, "failed to read the log store"), } } } impl std::error::Error for QueryError {} +/// Whether `dir` (a tenant's partition root) holds at least one +/// published `*.parquet` file anywhere beneath it. Recursive +/// because the data is nested `year=/month=/day=/hour=/`. Files +/// the writer hasn't committed (`*.parquet.tmp`) have extension +/// `tmp`, so they don't count — the poisoned-writer case we treat +/// as "empty", not error. +/// +/// A missing directory (`NotFound`) is "empty" (`Ok(false)`); any +/// *other* I/O error (permission denied, transient failure) is +/// propagated as [`QueryError::Storage`] rather than silently +/// masked as "no data" — a wrong zero-row answer is worse than a +/// surfaced error. +fn has_published_parquet(dir: &std::path::Path) -> Result { + let io_err = |op: &str, p: &std::path::Path, e: &std::io::Error| QueryError::Storage { + detail: format!("{op} {}: {e}", p.display()), + }; + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let entries = match std::fs::read_dir(&d) { + Ok(entries) => entries, + // The dir (or a subdir, lost to a concurrent + // housekeeping unlink) simply isn't there → not data, + // not an error. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(io_err("read_dir", &d, &e)), + }; + for entry in entries { + let entry = entry.map_err(|e| io_err("read_dir entry", &d, &e))?; + let path = entry.path(); + match entry.file_type() { + Ok(ft) if ft.is_dir() => stack.push(path), + Ok(_) if path.extension().is_some_and(|x| x == "parquet") => return Ok(true), + Ok(_) => {} + Err(e) => return Err(io_err("file_type", &path, &e)), + } + } + } + Ok(false) +} + +/// Map a `DataFusion` error to the Ourios-owned [`QueryError`] so +/// no `datafusion`/`arrow` type crosses the public boundary (§4.6). +// Takes the error by value so it drops in cleanly as +// `.map_err(storage_err)`, which hands an owned error. +#[allow(clippy::needless_pass_by_value)] +fn storage_err(e: DataFusionError) -> QueryError { + QueryError::Storage { + detail: e.to_string(), + } +} + /// The query engine. One per querier process; reads the RFC 0005 -/// Parquet store from object storage. Construction wiring (object -/// store handle, `DataFusion` session) lands with the execution -/// slice. -#[derive(Debug, Default)] -pub struct Querier {} +/// Parquet store rooted at `bucket_root` (the writer's +/// `/data/...` layout). +#[derive(Debug, Clone)] +pub struct Querier { + bucket_root: PathBuf, +} impl Querier { - /// Create a querier. (Configuration — object-store root, - /// session tuning — is added with the execution slice.) - #[must_use] - pub fn new() -> Self { - Self {} + /// Create a querier reading the RFC 0005 store under + /// `bucket_root` (the same root the `ourios-parquet` writer + /// writes `data/tenant_id=…/year=…/…` under). + pub fn new(bucket_root: impl Into) -> Self { + Self { + bucket_root: bucket_root.into(), + } } - /// Execute `request` and return the matching rows + pruning - /// stats. Lowers the query to a `DataFusion` `LogicalPlan`, - /// runs it against the RFC 0005 store with predicate - /// pushdown, and returns results without exposing `DataFusion` - /// (§4.6). + /// Execute `request` against the RFC 0005 store with predicate + /// pushdown and return the matching row count + pruning stats, + /// without exposing `DataFusion` (§4.6). + /// + /// Tenant isolation is structural: the listing table is rooted + /// at the request tenant's `data/tenant_id=/` directory, + /// so no other tenant's rows are reachable (RFC0007.5). A + /// tenant with no data on disk yields an empty result. /// /// # Errors /// /// See [`QueryError`]. - // `async` is part of the RFC 0007 §4.1 API contract (`DataFusion` - // execution is async); the red-gate stub has no `.await` yet, - // which is the only reason `clippy::unused_async` fires. - #[allow(clippy::unused_async)] - pub async fn run(&self, _request: QueryRequest) -> Result { - unimplemented!("RFC 0007 red gate — execution pending (§4; blocked on RFC 0002 DSL)"); + pub async fn run(&self, request: QueryRequest) -> Result { + let enc = percent_encode_tenant(request.tenant.as_str()); + let tenant_dir = self + .bucket_root + .join("data") + .join(format!("tenant_id={enc}")); + // No published `*.parquet` under the tenant dir ⇒ the + // tenant has nothing queryable ⇒ empty result (not an + // error). Covers both the missing-dir case and a dir that + // holds only `*.parquet.tmp` (a poisoned/crashed writer) or + // empty partition dirs — where `infer_schema` would + // otherwise error and wrongly fail the query. + if !has_published_parquet(&tenant_dir)? { + return Ok(QueryResult::default()); + } + + let ctx = SessionContext::new(); + // Build the table URL from the canonical absolute path, + // scheme-less with a trailing slash. DataFusion 53 treats + // an absolute filesystem path as local and URI-encodes it + // internally — so spaces / reserved characters in the + // bucket path are handled, unlike a hand-built `file://…` + // string. `canonicalize` is safe: we just confirmed the + // directory exists. The trailing slash marks it a + // directory (not a single object). + let abs = tenant_dir.canonicalize().map_err(|e| QueryError::Storage { + detail: format!("canonicalize {}: {e}", tenant_dir.display()), + })?; + let url = ListingTableUrl::parse(format!("{}/", abs.display())).map_err(storage_err)?; + // `year/month/day/hour` are path-only Hive partition cols + // (parsed from the directory names); `tenant_id` is *not* + // listed — relative to this tenant-scoped root it's a plain + // file column, and the rooting is what enforces isolation. + let options = ListingOptions::new(Arc::new(ParquetFormat::default())) + .with_file_extension(".parquet") + .with_table_partition_cols(vec![ + ("year".to_string(), DataType::Utf8), + ("month".to_string(), DataType::Utf8), + ("day".to_string(), DataType::Utf8), + ("hour".to_string(), DataType::Utf8), + ]); + let schema = options + .infer_schema(&ctx.state(), &url) + .await + .map_err(storage_err)?; + let config = ListingTableConfig::new(url) + .with_listing_options(options) + .with_schema(schema); + let table = ListingTable::try_new(config).map_err(storage_err)?; + ctx.register_table("logs", Arc::new(table)) + .map_err(storage_err)?; + + let mut df = ctx.table("logs").await.map_err(storage_err)?; + if let Some((start, end)) = request.time_range { + // `time_unix_nano` is Timestamp(Nanosecond, "UTC") + // (RFC 0005 schema); match the literal type exactly. + let to_ts = |v: u64| -> Result { + let ns = i64::try_from(v).map_err(|_| QueryError::InvalidQuery { + detail: format!("time bound {v} exceeds i64 nanoseconds"), + })?; + Ok(ScalarValue::TimestampNanosecond( + Some(ns), + Some("UTC".into()), + )) + }; + df = df + .filter( + col(columns::TIME_UNIX_NANO) + .gt_eq(lit(to_ts(start)?)) + .and(col(columns::TIME_UNIX_NANO).lt(lit(to_ts(end)?))), + ) + .map_err(storage_err)?; + } + if let Some(template_id) = request.template_id { + df = df + .filter(col(columns::TEMPLATE_ID).eq(lit(template_id))) + .map_err(storage_err)?; + } + + // `count()` aggregates without materialising the matched + // rows' columns — so the heavy `attributes` / `params` / + // `body` columns are never read for a count, and Parquet + // projection pushdown applies. (`collect()` would buffer + // every column of every match.) + let rows = df.count().await.map_err(storage_err)? as u64; + // QueryStats (row-group pruning / bytes) is slice 2 — it + // comes from the ParquetExec metrics, not the row count. + Ok(QueryResult { + rows, + stats: QueryStats::default(), + }) } } @@ -153,12 +329,15 @@ mod tests { .to_string(), "invalid query: bad filter", ); + // Storage Display is intentionally generic — the engine + // `detail` is NOT surfaced (it would leak DataFusion/SQL + // specifics, §4.6 / RFC0007.3). assert_eq!( QueryError::Storage { - detail: "s3 timeout".into(), + detail: "Error during planning: SQL ...".into(), } .to_string(), - "storage read failed: s3 timeout", + "failed to read the log store", ); } diff --git a/crates/ourios-querier/tests/acceptance.rs b/crates/ourios-querier/tests/acceptance.rs index da67a5a3b..c35d9b1b6 100644 --- a/crates/ourios-querier/tests/acceptance.rs +++ b/crates/ourios-querier/tests/acceptance.rs @@ -54,12 +54,5 @@ fn rfc0007_4_forward_compatible_reads() { unimplemented!("RFC0007.4 — assert unknown/missing columns honour RFC 0005 §3.9 defaults"); } -#[ignore = "RFC 0007 red gate — execution pending (RFC0007.5)"] -#[test] -fn rfc0007_5_tenant_isolation() { - // CLAUDE.md §3.7: a query for tenant T returns no row from - // another tenant, enforced at the partition-prune layer; a - // query without a tenant is a usage error (QueryError:: - // TenantRequired), not a cross-tenant scan. - unimplemented!("RFC0007.5 — assert no cross-tenant rows; tenantless query is a usage error"); -} +// RFC0007.5 (tenant isolation) is now a LIVE test — see +// `tests/execution.rs::rfc0007_5_tenant_isolation`. diff --git a/crates/ourios-querier/tests/execution.rs b/crates/ourios-querier/tests/execution.rs new file mode 100644 index 000000000..bf10a8eac --- /dev/null +++ b/crates/ourios-querier/tests/execution.rs @@ -0,0 +1,209 @@ +//! Execution slice 1 — `Querier::run` against a real RFC 0005 +//! Parquet store written by `ourios-parquet`. Covers the minimal +//! predicate set B1/B2 need (tenant + time range + template-exact) +//! and RFC0007.5 tenant isolation. Row-group pruning stats +//! (RFC0007.1) are slice 2. + +use std::collections::HashMap; +use std::path::Path; + +use ourios_core::audit::ParamType; +use ourios_core::record::{BodyKind, MinedRecord, Param}; +use ourios_core::tenant::TenantId; +use ourios_parquet::{PartitionKey, Writer}; +use ourios_querier::{Querier, QueryError, QueryRequest}; + +/// 2026-04-02T10:58:00 UTC — all test records sit in one hour +/// (one partition per tenant), matching the round-trip fixtures. +const TS0: u64 = 1_775_127_480_000_000_000; + +fn rec(tenant: &str, template_id: u64, ts_ns: u64) -> MinedRecord { + MinedRecord { + tenant_id: TenantId::new(tenant), + template_id, + template_version: 1, + severity_number: 9, + severity_text: Some("INFO".to_string()), + scope_name: Some("lib.cart".to_string()), + scope_version: Some("1.0.0".to_string()), + time_unix_nano: ts_ns, + observed_time_unix_nano: Some(ts_ns + 1_000), + attributes: Vec::new(), + dropped_attributes_count: 0, + resource_attributes: Vec::new(), + trace_id: None, + span_id: None, + flags: 0x01, + event_name: None, + body_kind: BodyKind::String, + params: vec![Param { + type_tag: ParamType::Num, + value: "42".to_string(), + }], + separators: vec![String::new(), " ".to_string()], + body: None, + confidence: 1.0, + lossy_flag: false, + } +} + +/// Write records into the RFC 0005 store under `bucket`, grouping +/// by partition the way the ingester would. +fn write_all(bucket: &Path, recs: &[MinedRecord]) { + let mut by_part: HashMap> = HashMap::new(); + for r in recs { + by_part + .entry(PartitionKey::derive(r).expect("derive partition")) + .or_default() + .push(r.clone()); + } + for (part, rs) in by_part { + let mut w = Writer::open(bucket, part).expect("open writer"); + w.append_records(&rs).expect("append"); + w.close().expect("close"); + } +} + +fn req(tenant: &str, time_range: Option<(u64, u64)>, template_id: Option) -> QueryRequest { + QueryRequest { + tenant: TenantId::new(tenant), + time_range, + template_id, + } +} + +/// Tenant "a": 3 rows (template 1 ×2, template 2 ×1); tenant "b": +/// 1 row. Exercises the full filter surface. +#[tokio::test] +async fn executes_and_counts_matching_rows() { + let bucket = tempfile::TempDir::new().expect("temp"); + let a = vec![ + rec("a", 1, TS0), + rec("a", 1, TS0 + 1_000_000), + rec("a", 2, TS0 + 2_000_000), + ]; + let b = vec![rec("b", 1, TS0)]; + write_all(bucket.path(), &a); + write_all(bucket.path(), &b); + + let q = Querier::new(bucket.path()); + + // All of tenant a. + let r = q.run(req("a", None, None)).await.expect("run"); + assert_eq!(r.rows, 3, "tenant a has 3 rows"); + + // Template-exact (B2 shape): template 1 only. + let r = q.run(req("a", None, Some(1))).await.expect("run"); + assert_eq!(r.rows, 2, "tenant a has 2 template-1 rows"); + + // Time range [TS0, TS0 + 1.5ms): the first two rows. + let r = q + .run(req("a", Some((TS0, TS0 + 1_500_000)), None)) + .await + .expect("run"); + assert_eq!(r.rows, 2, "two rows fall in the half-open window"); + + // Template + time combined: template 2 is at TS0+2ms, outside + // the window → zero. + let r = q + .run(req("a", Some((TS0, TS0 + 1_500_000)), Some(2))) + .await + .expect("run"); + assert_eq!(r.rows, 0); +} + +/// RFC0007.5 — tenant isolation. A query for one tenant never sees +/// another's rows (structural: the listing table is rooted at the +/// tenant's partition dir), and a tenant with no data is empty, +/// not an error. +#[tokio::test] +async fn rfc0007_5_tenant_isolation() { + let bucket = tempfile::TempDir::new().expect("temp"); + write_all( + bucket.path(), + &[rec("a", 1, TS0), rec("a", 1, TS0 + 1_000_000)], + ); + write_all(bucket.path(), &[rec("b", 1, TS0)]); + + let q = Querier::new(bucket.path()); + + let a = q.run(req("a", None, None)).await.expect("run a"); + assert_eq!(a.rows, 2, "tenant a sees only its 2 rows"); + + let b = q.run(req("b", None, None)).await.expect("run b"); + assert_eq!(b.rows, 1, "tenant b sees only its 1 row"); + + // A tenant that wrote nothing: empty result, not an error. + let none = q.run(req("ghost", None, None)).await.expect("run ghost"); + assert_eq!(none.rows, 0, "unknown tenant yields an empty result"); +} + +/// A tenant directory that exists but holds no *committed* +/// `*.parquet` (only an uncommitted `*.parquet.tmp`, e.g. a +/// crashed writer) is an empty result, not an error — the query +/// must not fail when schema inference would find nothing. +#[tokio::test] +async fn tenant_dir_without_committed_parquet_is_empty() { + let bucket = tempfile::TempDir::new().expect("temp"); + // Mimic a poisoned/crashed writer: the partition dir and a + // `.tmp` file exist, but nothing was committed to `.parquet`. + let part = bucket + .path() + .join("data/tenant_id=a/year=2026/month=04/day=02/hour=10"); + std::fs::create_dir_all(&part).expect("mkdir partition"); + std::fs::write( + part.join("01890000-0000-7000-8000-000000000000.parquet.tmp"), + b"partial", + ) + .expect("write tmp"); + + let q = Querier::new(bucket.path()); + let r = q + .run(req("a", None, None)) + .await + .expect("run must not error"); + assert_eq!( + r.rows, 0, + "uncommitted .tmp-only tenant dir is empty, not an error" + ); +} + +/// A `bucket_root` whose path contains a space still resolves — +/// the URL is built from the canonical path (`DataFusion` +/// URI-encodes it), not a raw `file://{display}` string that would +/// choke on the space. +#[tokio::test] +async fn bucket_path_with_spaces_resolves() { + let tmp = tempfile::TempDir::new().expect("temp"); + let bucket = tmp.path().join("ourios bucket with spaces"); + std::fs::create_dir_all(&bucket).expect("mkdir spaced bucket"); + write_all(&bucket, &[rec("a", 1, TS0), rec("a", 1, TS0 + 1_000_000)]); + + let q = Querier::new(&bucket); + let r = q + .run(req("a", None, None)) + .await + .expect("run over spaced path"); + assert_eq!(r.rows, 2, "spaced bucket path queries correctly"); +} + +/// A real I/O error reading the tenant directory surfaces as +/// `QueryError::Storage` — it is NOT silently masked as an empty +/// result (which would be a wrong zero-row answer). Induced +/// portably: place a regular *file* where the tenant directory is +/// expected, so `read_dir` fails with `ENOTDIR` (not `NotFound`). +#[tokio::test] +async fn read_dir_error_surfaces_as_storage_not_empty() { + let bucket = tempfile::TempDir::new().expect("temp"); + // tenant "x" → data/tenant_id=x; make it a file, not a dir. + let data = bucket.path().join("data"); + std::fs::create_dir_all(&data).expect("mkdir data"); + std::fs::write(data.join("tenant_id=x"), b"not a directory").expect("write file"); + + let q = Querier::new(bucket.path()); + let result = q.run(req("x", None, None)).await; + assert!( + matches!(result, Err(QueryError::Storage { .. })), + "a non-NotFound read_dir error must surface as Storage, got {result:?}", + ); +}