diff --git a/Cargo.lock b/Cargo.lock index 94550ec3d3..d8f5513b9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,9 +52,8 @@ dependencies = [ [[package]] name = "adbc_core" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b891479797b5588e320f7fd3caf15faba311cf8f8a76911195b6a3d55304eb" +version = "0.21.0" +source = "git+https://github.com/apache/arrow-adbc.git#d57cc5013f9e6a38f97c77c05149d9ee5669d46a" dependencies = [ "arrow-array", "arrow-schema", @@ -62,9 +61,8 @@ dependencies = [ [[package]] name = "adbc_ffi" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30e0f28a8363a76a8ec802934223ea5811ea658308c7403c8beb9aa86fa808f" +version = "0.21.0" +source = "git+https://github.com/apache/arrow-adbc.git#d57cc5013f9e6a38f97c77c05149d9ee5669d46a" dependencies = [ "adbc_core", "arrow-array", @@ -86,12 +84,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "ahash" version = "0.8.12" @@ -209,19 +201,20 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "apache-avro" -version = "0.17.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" +checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" dependencies = [ "bigdecimal", - "bzip2 0.4.4", + "bon", + "bzip2 0.6.0", "crc32fast", "digest", - "libflate", "log", + "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.8.5", + "rand 0.9.2", "regex-lite", "serde", "serde_bytes", @@ -229,8 +222,7 @@ dependencies = [ "snap", "strum", "strum_macros", - "thiserror 1.0.69", - "typed-builder", + "thiserror 2.0.17", "uuid", "xz2", "zstd", @@ -259,9 +251,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" dependencies = [ "arrow-arith", "arrow-array", @@ -280,9 +272,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ "arrow-array", "arrow-buffer", @@ -294,9 +286,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash", "arrow-buffer", @@ -305,15 +297,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "num", ] [[package]] name = "arrow-buffer" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" dependencies = [ "bytes", "half", @@ -322,9 +314,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" dependencies = [ "arrow-array", "arrow-buffer", @@ -343,9 +335,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" +checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" dependencies = [ "arrow-array", "arrow-cast", @@ -358,9 +350,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -370,14 +362,15 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", "lz4_flex", "zstd", @@ -385,9 +378,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" +checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" dependencies = [ "arrow-array", "arrow-buffer", @@ -407,9 +400,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ "arrow-array", "arrow-buffer", @@ -420,9 +413,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ "arrow-array", "arrow-buffer", @@ -433,9 +426,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" dependencies = [ "bitflags", "serde", @@ -444,9 +437,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash", "arrow-array", @@ -458,9 +451,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ "arrow-array", "arrow-buffer", @@ -1005,6 +998,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bon" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +dependencies = [ + "darling 0.21.3", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.106", +] + [[package]] name = "brotli" version = "8.0.2" @@ -1060,16 +1078,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - [[package]] name = "bzip2" version = "0.5.2" @@ -1329,15 +1337,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "core_extensions" version = "1.5.4" @@ -1507,8 +1506,18 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -1525,22 +1534,41 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.106", +] + [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core", + "darling_core 0.13.4", "quote", "syn 1.0.109", ] [[package]] -name = "dary_heap" -version = "0.3.8" +name = "darling_macro" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.106", +] [[package]] name = "dashmap" @@ -1558,8 +1586,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6759cf9ef57c5c469e4027ac4b4cfa746e06a0f5472c2b922b6a403c2a64c4" dependencies = [ "arrow", "arrow-ipc", @@ -1587,6 +1616,7 @@ dependencies = [ "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", @@ -1594,7 +1624,6 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "hex", "itertools 0.14.0", "log", "object_store", @@ -1613,8 +1642,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a1c48fc7e6d62590d45f7be7c531980b8ff091d1ab113a9ddf465bef41e4093" dependencies = [ "arrow", "async-trait", @@ -1638,8 +1668,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db1266da115de3ab0b2669fc027d96cf0ff777deb3216d52c74b528446ccdd6" dependencies = [ "arrow", "async-trait", @@ -1660,8 +1691,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4eb2a48ca10fa1e1a487a28a5bf080e31efac2d4bf12bb7e92c2d9ea4f35e5" dependencies = [ "ahash", "apache-avro", @@ -1671,7 +1703,6 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", - "hex", "indexmap", "libc", "log", @@ -1686,8 +1717,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0422ee64d5791599c46b786063e695f7699fadd3a12ad25038cb3094d05886a" dependencies = [ "futures", "log", @@ -1696,8 +1728,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904c2e1089b3ccf10786f2dae12bc560fda278e4194a8917c5844d2e8c212818" dependencies = [ "arrow", "async-compression", @@ -1710,6 +1743,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -1731,8 +1765,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1c87828da05c4115935af3394b27499cd2bd91bcb846a928209650627bf7f93" dependencies = [ "apache-avro", "arrow", @@ -1755,8 +1790,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8336a805c42ef4e359daaad142ddc53649f23c7e934c117d8516816afe6b7a3d" dependencies = [ "arrow", "async-trait", @@ -1779,8 +1815,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c691b1565e245ea369bc8418b472a75ea84c2ad2deb61b1521cfa38319a9cd47" dependencies = [ "arrow", "async-trait", @@ -1803,8 +1840,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f7576ceb5974c5f6874d7f2a5ebfeb58960a920da64017def849e0352fe2d8" dependencies = [ "arrow", "async-trait", @@ -1817,13 +1855,13 @@ dependencies = [ "datafusion-expr", "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", "futures", - "hex", "itertools 0.14.0", "log", "object_store", @@ -1835,15 +1873,18 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dde7c10244f3657fc01eef8247c0b2b20eae4cf6439a0ebb27322f32026d6b8" [[package]] name = "datafusion-execution" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5143fc795cef959b6d5271b2e8f1120382fe929fc4bd027c7d7b993f5352ef7e" dependencies = [ "arrow", + "async-trait", "dashmap", "datafusion-common", "datafusion-expr", @@ -1858,8 +1899,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e826296bc5f5d0af3e39c1af473d4091ac6a152a5be2f80c256f0182938428" dependencies = [ "arrow", "async-trait", @@ -1879,8 +1921,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9096732d0d8862d1950ca70324fe91f9dee3799eeb0db53ef452bdb573484db6" dependencies = [ "arrow", "datafusion-common", @@ -1891,8 +1934,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4bef25e2b86d9921f7a98b1a86bfb50cebe2fd97f3a9b96c85ce475e9ef78b0" dependencies = [ "abi_stable", "arrow", @@ -1912,8 +1956,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f362c78ac283e64fd3976e060c1a8a57d5f4dcf844a6b6bd2eb320640a1572e" dependencies = [ "arrow", "arrow-buffer", @@ -1940,8 +1985,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e2a80a80145a796ae3f02eb724ac516178556aec864fe89f6ab3741a4cd249" dependencies = [ "ahash", "arrow", @@ -1960,8 +2006,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dcca2fe7c33409e9ab3f950366aa4cba5db6175a09599fdb658ad9f2cc4296" dependencies = [ "ahash", "arrow", @@ -1972,8 +2019,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1b298733377f3ec8c2868c75b5555b15396d9c13e36c5fda28e80feee34e3ed" dependencies = [ "arrow", "arrow-ord", @@ -1993,8 +2041,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fa4a380ca362eb0fbd33093e8ca6b7a31057616c7e6ee999b87a4ad3c7c0b3f" dependencies = [ "arrow", "async-trait", @@ -2008,8 +2057,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9068fc85b8e187c706427794d79bb7ee91132b6b192cb7b18e650a5f7c5c1340" dependencies = [ "arrow", "datafusion-common", @@ -2025,8 +2075,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f80ec56e177d166269556649be817a382a374642872df4ca48cf9be3d09b3a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2034,18 +2085,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4868fe261ba01e462033eff141e90453b7630722cad6420fddd81ebb786f6e2" dependencies = [ - "datafusion-doc", + "datafusion-expr", "quote", "syn 2.0.106", ] [[package]] name = "datafusion-optimizer" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ed8c51b5c37c057e5c7d5945ed807f1cecfba003bdb1a4c3036595dda287c7" dependencies = [ "arrow", "chrono", @@ -2063,8 +2116,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678f5734147446e1adbee63be4b244c8f0e9cbd5c41525004ace3730190d03e" dependencies = [ "ahash", "arrow", @@ -2078,14 +2132,31 @@ dependencies = [ "indexmap", "itertools 0.14.0", "log", + "parking_lot", "paste", "petgraph", ] +[[package]] +name = "datafusion-physical-expr-adapter" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086877d4eca538e9cd1f28b917db0036efe0ad8b4fb7c702f520510672032c8d" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + [[package]] name = "datafusion-physical-expr-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5c5d17f6a4f28f9849ee3449bb9b83406a718e4275c218bf37ca247ee123779" dependencies = [ "ahash", "arrow", @@ -2097,8 +2168,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9fb8b3fba2634d444e0177862797dc1231e0e20bc4db291a15d39c0d4136c3" dependencies = [ "arrow", "datafusion-common", @@ -2116,8 +2188,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5086cb2e579270173ff0eb38d60ba2a081f1d422a743fa673f6096920950eb5" dependencies = [ "ahash", "arrow", @@ -2129,6 +2202,7 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -2145,8 +2219,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87c686bfd29ec5362fe229247ef03a0beb063b50e307bf72d0f1a80b9d90f8b8" dependencies = [ "arrow", "chrono", @@ -2160,8 +2235,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1972d37680d48d4f6167b535e0a23ea9f814a21e1359d0bd5c30d1345b95aef9" dependencies = [ "arrow", "datafusion-common", @@ -2170,8 +2246,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f84b866d906118c320459f30385048aeedbe36ac06973d3e4fa0cc5d60d722c" dependencies = [ "arrow", "arrow-schema", @@ -2187,8 +2264,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3820062b9dd2846954eeb844ff9fe3662977b7d2d74947647c779fabfa502508" dependencies = [ "arrow", "async-trait", @@ -2210,8 +2288,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "49.0.2" -source = "git+https://github.com/paleolimbot/datafusion.git?branch=local-49-with-patch#697caabb6da842f6a561b0bebdd7847eef311153" +version = "50.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375232baa851b2e9d09fcbe8906141a0ec6e0e058addc5565e0d3d790bb9d51d" dependencies = [ "arrow", "bigdecimal", @@ -2278,9 +2357,9 @@ dependencies = [ [[package]] name = "document-features" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ "litrs", ] @@ -3370,30 +3449,6 @@ version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" -[[package]] -name = "libflate" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" -dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", -] - [[package]] name = "libloading" version = "0.7.4" @@ -3483,9 +3538,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "litrs" -version = "0.4.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lock_api" @@ -3848,9 +3903,9 @@ dependencies = [ [[package]] name = "parquet" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" dependencies = [ "ahash", "arrow-array", @@ -4493,12 +4548,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "robust" version = "1.2.0" @@ -4995,6 +5044,7 @@ dependencies = [ "datafusion", "datafusion-catalog", "datafusion-common", + "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", @@ -5329,7 +5379,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn 1.0.109", @@ -5421,9 +5471,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.55.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" dependencies = [ "log", "recursive", @@ -5474,20 +5524,19 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck", "proc-macro2", "quote", - "rustversion", "syn 2.0.106", ] @@ -5896,26 +5945,6 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" -[[package]] -name = "typed-builder" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "typenum" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index f314e53917..84b19f58db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,30 +56,31 @@ rust-version = "1.82" [workspace.dependencies] approx = "0.5" abi_stable = "0.11.3" -adbc_core = "0.20.0" -adbc_ffi = "0.20.0" +adbc_core = ">=0.20.0" +adbc_ffi = ">=0.20.0" lru = "0.12" -arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } -arrow-array = { version = "55.1.0" } -arrow-cast = { version = "55.1.0" } -arrow-data = { version = "55.1.0" } -arrow-json = { version = "55.1.0" } -arrow-schema = { version = "55.1.0" } +arrow = { version = "56.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } +arrow-array = { version = "56.0.0" } +arrow-cast = { version = "56.0.0" } +arrow-data = { version = "56.0.0" } +arrow-json = { version = "56.0.0" } +arrow-schema = { version = "56.0.0" } async-trait = { version = "0.1.87" } bytes = "1.10" byteorder = "1" chrono = { version = "0.4.38", default-features = false } -comfy-table = { version = "7.0" } +comfy-table = { version = "7.1.4" } criterion = { version = "0.5", features = ["html_reports"] } -datafusion = { version = "49.0.0", default-features = false } -datafusion-catalog = { version = "49.0.0" } -datafusion-common = { version = "49.0.0", default-features = false } -datafusion-common-runtime = { version = "49.0.0", default-features = false } -datafusion-execution = { version = "49.0.0", default-features = false } -datafusion-expr = { version = "49.0.0" } -datafusion-ffi = { version = "49.0.0" } -datafusion-physical-expr = { version = "49.0.0" } -datafusion-physical-plan = { version = "49.0.0" } +datafusion = { version = "50.2.0", default-features = false } +datafusion-catalog = { version = "50.2.0" } +datafusion-common = { version = "50.2.0", default-features = false } +datafusion-common-runtime = { version = "50.2.0", default-features = false } +datafusion-datasource-parquet = { version = "50.2.0" } +datafusion-execution = { version = "50.2.0", default-features = false } +datafusion-expr = { version = "50.2.0" } +datafusion-ffi = { version = "50.2.0" } +datafusion-physical-expr = { version = "50.2.0" } +datafusion-physical-plan = { version = "50.2.0" } dirs = "6.0.0" env_logger = "0.11" futures = { version = "0.3" } @@ -102,7 +103,7 @@ wkb = "0.9.1" wkt = "0.14.0" parking_lot = "0.12" -parquet = { version = "55.1.0", default-features = false, features = [ +parquet = { version = "56.0.0", default-features = false, features = [ "arrow", "async", "object_store", @@ -119,16 +120,9 @@ tokio = { version = "1.44" } url = "2.5.4" [patch.crates-io] -# For R package build on Windows (can be removed when we upgrade to DataFusion main/51.0.0) -datafusion = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion" } -datafusion-catalog = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-catalog" } -datafusion-common = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common" } -datafusion-common-runtime = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common-runtime" } -datafusion-execution = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-execution" } -datafusion-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-expr" } -datafusion-ffi = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-ffi" } -datafusion-physical-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-expr" } -datafusion-physical-plan = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-plan" } +# Use main branch of arrow-adbc which supports Arrow 56.x (remove when 0.21.0 is released) +adbc_core = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_core" } +adbc_ffi = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_ffi" } # TODO: remove them once changes we made to geo-index and wkb crates are merged to upstream and released geo-index = { git = "https://github.com/wherobots/geo-index.git", branch = "main" } diff --git a/c/sedona-proj/src/st_transform.rs b/c/sedona-proj/src/st_transform.rs index 51a268245b..97f0781cf6 100644 --- a/c/sedona-proj/src/st_transform.rs +++ b/c/sedona-proj/src/st_transform.rs @@ -331,6 +331,7 @@ mod tests { use super::*; use arrow_array::ArrayRef; use arrow_schema::{DataType, Field}; + use datafusion_common::config::ConfigOptions; use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl}; use rstest::rstest; use sedona_expr::scalar_udf::SedonaScalarUDF; @@ -608,6 +609,7 @@ mod tests { arg_fields: arg_fields.to_vec(), number_rows: row_count, return_field, + config_options: Arc::new(ConfigOptions::default()), }; let value = udf.invoke_with_args(args)?; diff --git a/python/sedonadb/python/sedonadb/dataframe.py b/python/sedonadb/python/sedonadb/dataframe.py index d1c3288bf6..f1b641b971 100644 --- a/python/sedonadb/python/sedonadb/dataframe.py +++ b/python/sedonadb/python/sedonadb/dataframe.py @@ -441,7 +441,7 @@ def explain( │ utf8 ┆ utf8 │ ╞═══════════════╪═════════════════════════════════╡ │ logical_plan ┆ Projection: Int64(1) AS one │ - │ ┆ EmptyRelation │ + │ ┆ EmptyRelation: rows=1 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ physical_plan ┆ ProjectionExec: expr=[1 as one] │ │ ┆ PlaceholderRowExec │ diff --git a/python/sedonadb/tests/test_dataframe.py b/python/sedonadb/tests/test_dataframe.py index acf27ecd3f..bb3ca7828e 100644 --- a/python/sedonadb/tests/test_dataframe.py +++ b/python/sedonadb/tests/test_dataframe.py @@ -444,7 +444,7 @@ def test_show_explained(con, capsys): │ utf8 ┆ utf8 │ ╞═══════════════╪═════════════════════════════════╡ │ logical_plan ┆ Projection: Int64(1) AS one │ -│ ┆ EmptyRelation │ +│ ┆ EmptyRelation: rows=1 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ physical_plan ┆ ProjectionExec: expr=[1 as one] │ │ ┆ PlaceholderRowExec │ @@ -462,7 +462,7 @@ def test_explain(con, capsys): │ utf8 ┆ utf8 │ ╞═══════════════╪═════════════════════════════════╡ │ logical_plan ┆ Projection: Int64(1) AS one │ -│ ┆ EmptyRelation │ +│ ┆ EmptyRelation: rows=1 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ physical_plan ┆ ProjectionExec: expr=[1 as one] │ │ ┆ PlaceholderRowExec │ diff --git a/rust/sedona-expr/src/aggregate_udf.rs b/rust/sedona-expr/src/aggregate_udf.rs index 9d04b5f726..3ec999d00a 100644 --- a/rust/sedona-expr/src/aggregate_udf.rs +++ b/rust/sedona-expr/src/aggregate_udf.rs @@ -41,6 +41,20 @@ pub struct SedonaAggregateUDF { documentation: Option, } +impl PartialEq for SedonaAggregateUDF { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for SedonaAggregateUDF {} + +impl std::hash::Hash for SedonaAggregateUDF { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + impl SedonaAggregateUDF { /// Create a new SedonaAggregateUDF pub fn new( diff --git a/rust/sedona-expr/src/scalar_udf.rs b/rust/sedona-expr/src/scalar_udf.rs index 7f4c187d52..225d0d926b 100644 --- a/rust/sedona-expr/src/scalar_udf.rs +++ b/rust/sedona-expr/src/scalar_udf.rs @@ -41,6 +41,20 @@ pub struct SedonaScalarUDF { aliases: Vec, } +impl PartialEq for SedonaScalarUDF { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for SedonaScalarUDF {} + +impl std::hash::Hash for SedonaScalarUDF { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + /// User-defined function implementation /// /// A `SedonaScalarUdf` is comprised of one or more kernels, to which it dispatches diff --git a/rust/sedona-expr/src/spatial_filter.rs b/rust/sedona-expr/src/spatial_filter.rs index 246362b401..83e314eb8c 100644 --- a/rust/sedona-expr/src/spatial_filter.rs +++ b/rust/sedona-expr/src/spatial_filter.rs @@ -402,6 +402,7 @@ fn parse_args(args: &[Arc]) -> Vec> { #[cfg(test)] mod test { use arrow_schema::{DataType, Field}; + use datafusion_common::config::ConfigOptions; use datafusion_expr::{ScalarUDF, Signature, SimpleScalarUDF, Volatility}; use rstest::rstest; use sedona_geometry::{bounding_box::BoundingBox, interval::Interval}; @@ -589,6 +590,7 @@ mod test { Arc::new(unrelated), vec![], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_no_args).unwrap(), @@ -614,6 +616,7 @@ mod test { Arc::new(func.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -627,6 +630,7 @@ mod test { Arc::new(func), vec![literal.clone(), column.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -651,6 +655,7 @@ mod test { Arc::new(func.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -664,6 +669,7 @@ mod test { Arc::new(func), vec![literal.clone(), column.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -690,6 +696,7 @@ mod test { Arc::new(func.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -703,6 +710,7 @@ mod test { Arc::new(func), vec![literal.clone(), column.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -730,6 +738,7 @@ mod test { Arc::new(func.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -744,6 +753,7 @@ mod test { Arc::new(func), vec![literal.clone(), column.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap(); assert!( @@ -770,6 +780,7 @@ mod test { Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), distance_literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -783,6 +794,7 @@ mod test { Arc::new(st_dwithin), vec![literal.clone(), column.clone(), distance_literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap(); assert!( @@ -797,6 +809,7 @@ mod test { Arc::new(st_distance.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let comparison_expr: Arc = Arc::new(BinaryExpr::new( distance_expr.clone(), @@ -830,6 +843,7 @@ mod test { Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), negative_distance], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -845,6 +859,7 @@ mod test { Arc::new(st_dwithin), vec![column.clone(), literal.clone(), nan_distance], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_nan = SpatialFilter::try_from_expr(&dwithin_expr_nan).unwrap(); assert!( @@ -878,6 +893,7 @@ mod test { Arc::new(st_intersects.clone()), vec![], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(SpatialFilter::try_from_expr(&expr_no_args) .unwrap_err() @@ -890,6 +906,7 @@ mod test { Arc::new(st_intersects.clone()), vec![literal.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(), @@ -926,6 +943,7 @@ mod test { Arc::new(func.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!( @@ -952,6 +970,7 @@ mod test { Arc::new(st_dwithin.clone()), vec![column.clone(), literal.clone(), distance_literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap(); assert!( @@ -965,6 +984,7 @@ mod test { Arc::new(st_dwithin), vec![literal.clone(), column.clone(), distance_literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap(); assert!( @@ -979,6 +999,7 @@ mod test { Arc::new(st_distance.clone()), vec![column.clone(), literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let comparison_expr: Arc = Arc::new(BinaryExpr::new( distance_expr.clone(), @@ -1014,6 +1035,7 @@ mod test { Arc::new(has_z.clone()), vec![column.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); let predicate = SpatialFilter::try_from_expr(&expr).unwrap(); assert!(matches!(predicate, SpatialFilter::HasZ(_))); @@ -1029,6 +1051,7 @@ mod test { Arc::new(has_z.clone()), vec![], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(SpatialFilter::try_from_expr(&expr_no_args) .unwrap_err() @@ -1041,6 +1064,7 @@ mod test { Arc::new(has_z.clone()), vec![literal.clone()], Arc::new(Field::new("", DataType::Boolean, true)), + Arc::new(ConfigOptions::default()), )); assert!(matches!( SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(), diff --git a/rust/sedona-functions/src/st_setsrid.rs b/rust/sedona-functions/src/st_setsrid.rs index 007b21b82a..68d916eb40 100644 --- a/rust/sedona-functions/src/st_setsrid.rs +++ b/rust/sedona-functions/src/st_setsrid.rs @@ -232,6 +232,7 @@ mod test { use std::rc::Rc; use arrow_schema::Field; + use datafusion_common::config::ConfigOptions; use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; use sedona_geometry::{error::SedonaGeometryError, transform::CrsTransform}; use sedona_schema::{ @@ -374,6 +375,7 @@ mod test { arg_fields, number_rows: 1, return_field, + config_options: Arc::new(ConfigOptions::default()), }; let value = udf.invoke_with_args(args)?; diff --git a/rust/sedona-geoparquet/Cargo.toml b/rust/sedona-geoparquet/Cargo.toml index 0a5110f37e..cba3bd52cc 100644 --- a/rust/sedona-geoparquet/Cargo.toml +++ b/rust/sedona-geoparquet/Cargo.toml @@ -46,6 +46,7 @@ chrono = { workspace = true } datafusion = { workspace = true, features = ["parquet"] } datafusion-catalog = { workspace = true } datafusion-common = { workspace = true } +datafusion-datasource-parquet = { workspace = true } datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } datafusion-physical-expr = { workspace = true } diff --git a/rust/sedona-geoparquet/src/file_opener.rs b/rust/sedona-geoparquet/src/file_opener.rs index a71c95454c..183fe0a847 100644 --- a/rust/sedona-geoparquet/src/file_opener.rs +++ b/rust/sedona-geoparquet/src/file_opener.rs @@ -18,11 +18,11 @@ use std::{collections::HashMap, sync::Arc}; use arrow_schema::SchemaRef; use datafusion::datasource::{ - file_format::parquet::fetch_parquet_metadata, listing::PartitionedFile, physical_plan::{parquet::ParquetAccessPlan, FileMeta, FileOpenFuture, FileOpener}, }; use datafusion_common::Result; +use datafusion_datasource_parquet::metadata::DFParquetMetadata; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet, MetricBuilder}; use object_store::ObjectStore; @@ -113,13 +113,11 @@ impl FileOpener for GeoParquetFileOpener { let self_clone = self.clone(); Ok(Box::pin(async move { - let parquet_metadata = fetch_parquet_metadata( - &self_clone.object_store, - &file_meta.object_meta, - self_clone.metadata_size_hint, - None, - ) - .await?; + let parquet_metadata = + DFParquetMetadata::new(&self_clone.object_store, &file_meta.object_meta) + .with_metadata_size_hint(self_clone.metadata_size_hint) + .fetch_metadata() + .await?; let mut access_plan = ParquetAccessPlan::new_all(parquet_metadata.num_row_groups()); diff --git a/rust/sedona-geoparquet/src/format.rs b/rust/sedona-geoparquet/src/format.rs index 8ff87177f0..75eafb9d78 100644 --- a/rust/sedona-geoparquet/src/format.rs +++ b/rust/sedona-geoparquet/src/format.rs @@ -24,7 +24,7 @@ use datafusion::{ datasource::{ file_format::{ file_compression_type::FileCompressionType, - parquet::{fetch_parquet_metadata, ParquetFormat, ParquetFormatFactory}, + parquet::{ParquetFormat, ParquetFormatFactory}, FileFormat, FileFormatFactory, }, physical_plan::{ @@ -34,6 +34,7 @@ use datafusion::{ }; use datafusion_catalog::{memory::DataSourceExec, Session}; use datafusion_common::{plan_err, GetExt, Result, Statistics}; +use datafusion_datasource_parquet::metadata::DFParquetMetadata; use datafusion_physical_expr::{LexRequirement, PhysicalExpr}; use datafusion_physical_plan::{ filter_pushdown::FilterPushdownPropagation, metrics::ExecutionPlanMetricsSet, ExecutionPlan, @@ -182,13 +183,11 @@ impl FileFormat for GeoParquetFormat { // copy more ParquetFormat code. It may be that caching at the object // store level is the way to go here. let metadatas: Vec<_> = futures::stream::iter(objects) - .map(|object| { - fetch_parquet_metadata( - store.as_ref(), - object, - self.inner().metadata_size_hint(), - None, - ) + .map(|object| async move { + DFParquetMetadata::new(store.as_ref(), object) + .with_metadata_size_hint(self.inner().metadata_size_hint()) + .fetch_metadata() + .await }) .boxed() // Workaround https://github.com/rust-lang/rust/issues/64552 .buffered(state.config_options().execution.meta_fetch_concurrency) diff --git a/rust/sedona-geoparquet/src/writer.rs b/rust/sedona-geoparquet/src/writer.rs index ef741e1d8b..2b2ec9a4f6 100644 --- a/rust/sedona-geoparquet/src/writer.rs +++ b/rust/sedona-geoparquet/src/writer.rs @@ -28,7 +28,9 @@ use datafusion::{ file_format::parquet::ParquetSink, physical_plan::FileSinkConfig, sink::DataSinkExec, }, }; -use datafusion_common::{exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result}; +use datafusion_common::{ + config::ConfigOptions, exec_datafusion_err, exec_err, not_impl_err, DataFusionError, Result, +}; use datafusion_expr::{dml::InsertOp, ColumnarValue, ScalarUDF, Volatility}; use datafusion_physical_expr::{ expressions::Column, LexRequirement, PhysicalExpr, ScalarFunctionExpr, @@ -225,11 +227,14 @@ fn project_bboxes( column.return_field(&input_schema)?.as_ref(), )?) { let bbox_field_name = bbox_column_name(f.name()); + // TODO: Pipe actual ConfigOptions from session instead of using defaults + // See: https://github.com/apache/sedona-db/issues/248 let expr = Arc::new(ScalarFunctionExpr::new( bbox_udf_name, bbox_udf.clone(), vec![column], Arc::new(Field::new("", bbox_type(), true)), + Arc::new(ConfigOptions::default()), )); bbox_exprs.insert(i, (expr, bbox_field_name.clone())); diff --git a/rust/sedona-schema/src/schema.rs b/rust/sedona-schema/src/schema.rs index c3732c2696..9ce1d3a40f 100644 --- a/rust/sedona-schema/src/schema.rs +++ b/rust/sedona-schema/src/schema.rs @@ -41,15 +41,18 @@ pub trait SedonaSchema { impl SedonaSchema for DFSchema { fn sedona_types(&self) -> impl ExactSizeIterator> { - self.as_arrow().sedona_types() + let arrow_schema = self.as_arrow(); + ::sedona_types(arrow_schema) } fn geometry_column_indices(&self) -> Result> { - self.as_arrow().geometry_column_indices() + let arrow_schema = self.as_arrow(); + ::geometry_column_indices(arrow_schema) } fn primary_geometry_column_index(&self) -> Result> { - self.as_arrow().primary_geometry_column_index() + let arrow_schema = self.as_arrow(); + ::primary_geometry_column_index(arrow_schema) } } diff --git a/rust/sedona-spatial-join/src/optimizer.rs b/rust/sedona-spatial-join/src/optimizer.rs index e996906c96..bd01821b1e 100644 --- a/rust/sedona-spatial-join/src/optimizer.rs +++ b/rust/sedona-spatial-join/src/optimizer.rs @@ -133,7 +133,8 @@ impl OptimizerRule for SpatialJoinOptimizer { plan: LogicalPlan, config: &dyn OptimizerConfig, ) -> Result> { - let Some(extension) = config.options().extensions.get::() else { + let options = config.options(); + let Some(extension) = options.extensions.get::() else { return Ok(Transformed::no(plan)); }; if !extension.spatial_join.enable { @@ -1161,11 +1162,14 @@ mod tests { ) -> Arc { let return_type = udf.return_type(&[]).unwrap(); let field = Arc::new(arrow::datatypes::Field::new("result", return_type, false)); + // TODO: Pipe actual ConfigOptions from session instead of using defaults + // See: https://github.com/apache/sedona-db/issues/248 Arc::new(ScalarFunctionExpr::new( udf.name(), Arc::clone(&udf), args, field, + Arc::new(ConfigOptions::default()), )) } diff --git a/rust/sedona-testing/src/testers.rs b/rust/sedona-testing/src/testers.rs index a97bbeee7a..b6580a0e75 100644 --- a/rust/sedona-testing/src/testers.rs +++ b/rust/sedona-testing/src/testers.rs @@ -18,7 +18,7 @@ use std::{iter::zip, sync::Arc}; use arrow_array::{ArrayRef, RecordBatch}; use arrow_schema::{FieldRef, Schema}; -use datafusion_common::{Result, ScalarValue}; +use datafusion_common::{config::ConfigOptions, Result, ScalarValue}; use datafusion_expr::{ function::{AccumulatorArgs, StateFieldsArgs}, Accumulator, AggregateUDF, ColumnarValue, Expr, Literal, ReturnFieldArgs, ScalarFunctionArgs, @@ -404,6 +404,9 @@ impl ScalarUdfTester { arg_fields: self.arg_fields(), number_rows, return_field: self.return_type()?.to_storage_field("", true)?.into(), + // TODO: Consider piping actual ConfigOptions for more realistic testing + // See: https://github.com/apache/sedona-db/issues/248 + config_options: Arc::new(ConfigOptions::default()), }; self.udf.invoke_with_args(args) diff --git a/rust/sedona/src/context.rs b/rust/sedona/src/context.rs index ded65aa486..744dca22bd 100644 --- a/rust/sedona/src/context.rs +++ b/rust/sedona/src/context.rs @@ -539,7 +539,7 @@ mod tests { "| plan_type | plan |", "+---------------+---------------------------------+", "| logical_plan | Projection: Int64(1) AS one |", - "| | EmptyRelation |", + "| | EmptyRelation: rows=1 |", "| physical_plan | ProjectionExec: expr=[1 as one] |", "| | PlaceholderRowExec |", "| | |", diff --git a/rust/sedona/src/ffi.rs b/rust/sedona/src/ffi.rs index c4b0e1b16c..e1cb1f7bf6 100644 --- a/rust/sedona/src/ffi.rs +++ b/rust/sedona/src/ffi.rs @@ -19,7 +19,7 @@ use std::{any::Any, sync::Arc}; use abi_stable::StableAbi; use arrow_schema::{DataType, Field, FieldRef, Schema}; use datafusion::physical_plan::{expressions::Column, PhysicalExpr}; -use datafusion_common::{DataFusionError, Result, ScalarValue}; +use datafusion_common::{config::ConfigOptions, DataFusionError, Result, ScalarValue}; use datafusion_expr::{ function::{AccumulatorArgs, StateFieldsArgs}, Accumulator, AggregateUDF, AggregateUDFImpl, ColumnarValue, ReturnFieldArgs, @@ -89,6 +89,20 @@ struct ExportedScalarKernel { sedona_impl: ScalarKernelRef, } +impl PartialEq for ExportedScalarKernel { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for ExportedScalarKernel {} + +impl std::hash::Hash for ExportedScalarKernel { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + impl From for ExportedScalarKernel { fn from(value: ScalarKernelRef) -> Self { Self { @@ -196,6 +210,7 @@ impl SedonaScalarKernel for ImportedScalarKernel { number_rows: arg_rows.unwrap_or(1), // Wrapper code on the other side of this doesn't use this value return_field: Field::new("", DataType::Null, true).into(), + config_options: Arc::new(ConfigOptions::default()), }; // DataFusion's FFI_ScalarUDF always returns array output but @@ -273,6 +288,20 @@ struct ExportedSedonaAccumulator { sedona_impl: SedonaAccumulatorRef, } +impl PartialEq for ExportedSedonaAccumulator { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + +impl Eq for ExportedSedonaAccumulator {} + +impl std::hash::Hash for ExportedSedonaAccumulator { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + impl From for ExportedSedonaAccumulator { fn from(value: SedonaAccumulatorRef) -> Self { Self { diff --git a/rust/sedona/src/random_geometry_provider.rs b/rust/sedona/src/random_geometry_provider.rs index c98f415dab..00b67dc9bc 100644 --- a/rust/sedona/src/random_geometry_provider.rs +++ b/rust/sedona/src/random_geometry_provider.rs @@ -199,7 +199,7 @@ impl TableProvider for RandomGeometryProvider { // We're required to handle the projection or we'll get an execution error if let Some(projection) = projection { let schema = self.schema(); - let exprs = projection + let exprs: Vec<_> = projection .iter() .map(|index| -> (Arc, String) { let name = schema.field(*index).name(); diff --git a/rust/sedona/src/show.rs b/rust/sedona/src/show.rs index f1ac3621fa..4276a11980 100644 --- a/rust/sedona/src/show.rs +++ b/rust/sedona/src/show.rs @@ -21,7 +21,7 @@ use comfy_table::{Cell, CellAlignment, ColumnConstraint, ContentArrangement, Row use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; use datafusion::error::Result; use datafusion_common::format::DEFAULT_FORMAT_OPTIONS; -use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_common::{config::ConfigOptions, DataFusionError, ScalarValue}; use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; use sedona_expr::scalar_udf::SedonaScalarUDF; use sedona_schema::{datatypes::SedonaType, matchers::ArgMatcher}; @@ -495,6 +495,9 @@ impl DisplayColumn { arg_fields, number_rows: array.len(), return_field, + // TODO: Pipe actual ConfigOptions from SedonaContext instead of using defaults + // See: https://github.com/apache/sedona-db/issues/248 + config_options: Arc::new(ConfigOptions::default()), }; let format_proxy_value = format_udf.invoke_with_args(args)?;