chore: Upgrade Datafusion (v50) and Arrow (v56) dependencies

Cargo.toml

@@ -56,30 +56,30 @@ rust-version = "1.82"
 [workspace.dependencies]
 approx = "0.5"
 abi_stable = "0.11.3"
-adbc_core = "0.20.0"
-adbc_ffi = "0.20.0"
+adbc_core = ">=0.20.0"
+adbc_ffi = ">=0.20.0"
 lru = "0.12"
-arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { version = "55.1.0" }
-arrow-cast = { version = "55.1.0" }
-arrow-data = { version = "55.1.0" }
-arrow-json = { version = "55.1.0" }
-arrow-schema = { version = "55.1.0" }
+arrow = { version = "56.0.0", features = ["prettyprint", "ffi", "chrono-tz"] }
+arrow-array = { version = "56.0.0" }
+arrow-cast = { version = "56.0.0" }
+arrow-data = { version = "56.0.0" }
+arrow-json = { version = "56.0.0" }
+arrow-schema = { version = "56.0.0" }
 async-trait = { version = "0.1.87" }
 bytes = "1.10"
 byteorder = "1"
 chrono = { version = "0.4.38", default-features = false }
 comfy-table = { version = "7.0" }
 criterion = { version = "0.5", features = ["html_reports"] }
-datafusion = { version = "49.0.0", default-features = false }
-datafusion-catalog = { version = "49.0.0" }
-datafusion-common = { version = "49.0.0", default-features = false }
-datafusion-common-runtime = { version = "49.0.0", default-features = false }
-datafusion-execution = { version = "49.0.0", default-features = false }
-datafusion-expr = { version = "49.0.0" }
-datafusion-ffi = {  version = "49.0.0" }
-datafusion-physical-expr = { version = "49.0.0" }
-datafusion-physical-plan = { version = "49.0.0" }
+datafusion = { version = "50.2.0", default-features = false }
+datafusion-catalog = { version = "50.2.0" }
+datafusion-common = { version = "50.2.0", default-features = false }
+datafusion-common-runtime = { version = "50.2.0", default-features = false }
+datafusion-execution = { version = "50.2.0", default-features = false }
+datafusion-expr = { version = "50.2.0" }
+datafusion-ffi = {  version = "50.2.0" }
+datafusion-physical-expr = { version = "50.2.0" }
+datafusion-physical-plan = { version = "50.2.0" }
 dirs = "6.0.0"
 env_logger = "0.11"
 futures = { version = "0.3" }
@@ -102,7 +102,7 @@ wkb = "0.9.1"
 wkt = "0.14.0"
 
 parking_lot = "0.12"
-parquet = { version = "55.1.0", default-features = false, features = [
+parquet = { version = "56.0.0", default-features = false, features = [
     "arrow",
     "async",
     "object_store",
@@ -119,16 +119,9 @@ tokio = { version = "1.44" }
 url = "2.5.4"
 
 [patch.crates-io]
-# For R package build on Windows (can be removed when we upgrade to DataFusion main/51.0.0)
-datafusion = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion" }
-datafusion-catalog = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-catalog" }
-datafusion-common = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common" }
-datafusion-common-runtime = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common-runtime" }
-datafusion-execution = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-execution" }
-datafusion-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-expr" }
-datafusion-ffi = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-ffi" }
-datafusion-physical-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-expr" }
-datafusion-physical-plan = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-plan" }
+# Use main branch of arrow-adbc which supports Arrow 56.x (remove when 0.21.0 is released)
+adbc_core = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_core" }
+adbc_ffi = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_ffi" }
 
 # TODO: remove them once changes we made to geo-index and wkb crates are merged to upstream and released
 geo-index = { git = "https://github.com/wherobots/geo-index.git", branch = "main" }

c/sedona-proj/src/st_transform.rs

@@ -331,6 +331,7 @@ mod tests {
     use super::*;
     use arrow_array::ArrayRef;
     use arrow_schema::{DataType, Field};
+    use datafusion_common::config::ConfigOptions;
     use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl};
     use rstest::rstest;
     use sedona_expr::scalar_udf::SedonaScalarUDF;
@@ -608,6 +609,7 @@ mod tests {
             arg_fields: arg_fields.to_vec(),
             number_rows: row_count,
             return_field,
+            config_options: Arc::new(ConfigOptions::default()),
         };
 
         let value = udf.invoke_with_args(args)?;

python/sedonadb/python/sedonadb/dataframe.py

@@ -441,7 +441,7 @@ def explain(
             │      utf8     ┆               utf8              │
             ╞═══════════════╪═════════════════════════════════╡
             │ logical_plan  ┆ Projection: Int64(1) AS one     │
-            │               ┆   EmptyRelation                 │
+            │               ┆   EmptyRelation: rows=1         │
             ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
             │ physical_plan ┆ ProjectionExec: expr=[1 as one] │
             │               ┆   PlaceholderRowExec            │

python/sedonadb/tests/test_dataframe.py

@@ -444,7 +444,7 @@ def test_show_explained(con, capsys):
 │      utf8     ┆               utf8              │
 ╞═══════════════╪═════════════════════════════════╡
 │ logical_plan  ┆ Projection: Int64(1) AS one     │
-│               ┆   EmptyRelation                 │
+│               ┆   EmptyRelation: rows=1         │
 ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
 │ physical_plan ┆ ProjectionExec: expr=[1 as one] │
 │               ┆   PlaceholderRowExec            │
@@ -462,7 +462,7 @@ def test_explain(con, capsys):
 │      utf8     ┆               utf8              │
 ╞═══════════════╪═════════════════════════════════╡
 │ logical_plan  ┆ Projection: Int64(1) AS one     │
-│               ┆   EmptyRelation                 │
+│               ┆   EmptyRelation: rows=1         │
 ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
 │ physical_plan ┆ ProjectionExec: expr=[1 as one] │
 │               ┆   PlaceholderRowExec            │

rust/sedona-expr/src/aggregate_udf.rs

@@ -41,6 +41,20 @@ pub struct SedonaAggregateUDF {
     documentation: Option<Documentation>,
 }
 
+impl PartialEq for SedonaAggregateUDF {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+    }
+}
+
+impl Eq for SedonaAggregateUDF {}
+
+impl std::hash::Hash for SedonaAggregateUDF {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.name.hash(state);
+    }
+}
+
 impl SedonaAggregateUDF {
     /// Create a new SedonaAggregateUDF
     pub fn new(

rust/sedona-expr/src/scalar_udf.rs

@@ -41,6 +41,20 @@ pub struct SedonaScalarUDF {
     aliases: Vec<String>,
 }
 
+impl PartialEq for SedonaScalarUDF {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+    }
+}
+
+impl Eq for SedonaScalarUDF {}
+
+impl std::hash::Hash for SedonaScalarUDF {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.name.hash(state);
+    }
+}
+
 /// User-defined function implementation
 ///
 /// A `SedonaScalarUdf` is comprised of one or more kernels, to which it dispatches

rust/sedona-expr/src/spatial_filter.rs

@@ -402,6 +402,7 @@ fn parse_args(args: &[Arc<dyn PhysicalExpr>]) -> Vec<ArgRef<'_>> {
 #[cfg(test)]
 mod test {
     use arrow_schema::{DataType, Field};
+    use datafusion_common::config::ConfigOptions;
     use datafusion_expr::{ScalarUDF, Signature, SimpleScalarUDF, Volatility};
     use rstest::rstest;
     use sedona_geometry::{bounding_box::BoundingBox, interval::Interval};
@@ -589,6 +590,7 @@ mod test {
             Arc::new(unrelated),
             vec![],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         assert!(matches!(
             SpatialFilter::try_from_expr(&expr_no_args).unwrap(),
@@ -614,6 +616,7 @@ mod test {
             Arc::new(func.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
         assert!(
@@ -627,6 +630,7 @@ mod test {
             Arc::new(func),
             vec![literal.clone(), column.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
         assert!(
@@ -651,6 +655,7 @@ mod test {
             Arc::new(func.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
         assert!(
@@ -664,6 +669,7 @@ mod test {
             Arc::new(func),
             vec![literal.clone(), column.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
         assert!(
@@ -690,6 +696,7 @@ mod test {
             Arc::new(func.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
         assert!(
@@ -703,6 +710,7 @@ mod test {
             Arc::new(func),
             vec![literal.clone(), column.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
         assert!(
@@ -730,6 +738,7 @@ mod test {
             Arc::new(func.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
         assert!(
@@ -744,6 +753,7 @@ mod test {
             Arc::new(func),
             vec![literal.clone(), column.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
         assert!(
@@ -770,6 +780,7 @@ mod test {
             Arc::new(st_dwithin.clone()),
             vec![column.clone(), literal.clone(), distance_literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap();
         assert!(
@@ -783,6 +794,7 @@ mod test {
             Arc::new(st_dwithin),
             vec![literal.clone(), column.clone(), distance_literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap();
         assert!(
@@ -797,6 +809,7 @@ mod test {
             Arc::new(st_distance.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let comparison_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
             distance_expr.clone(),
@@ -830,6 +843,7 @@ mod test {
             Arc::new(st_dwithin.clone()),
             vec![column.clone(), literal.clone(), negative_distance],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap();
         assert!(
@@ -845,6 +859,7 @@ mod test {
             Arc::new(st_dwithin),
             vec![column.clone(), literal.clone(), nan_distance],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_nan = SpatialFilter::try_from_expr(&dwithin_expr_nan).unwrap();
         assert!(
@@ -878,6 +893,7 @@ mod test {
             Arc::new(st_intersects.clone()),
             vec![],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         assert!(SpatialFilter::try_from_expr(&expr_no_args)
             .unwrap_err()
@@ -890,6 +906,7 @@ mod test {
             Arc::new(st_intersects.clone()),
             vec![literal.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         assert!(matches!(
             SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(),
@@ -926,6 +943,7 @@ mod test {
             Arc::new(func.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
         assert!(
@@ -952,6 +970,7 @@ mod test {
             Arc::new(st_dwithin.clone()),
             vec![column.clone(), literal.clone(), distance_literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap();
         assert!(
@@ -965,6 +984,7 @@ mod test {
             Arc::new(st_dwithin),
             vec![literal.clone(), column.clone(), distance_literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap();
         assert!(
@@ -979,6 +999,7 @@ mod test {
             Arc::new(st_distance.clone()),
             vec![column.clone(), literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let comparison_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
             distance_expr.clone(),
@@ -1014,6 +1035,7 @@ mod test {
             Arc::new(has_z.clone()),
             vec![column.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
         assert!(matches!(predicate, SpatialFilter::HasZ(_)));
@@ -1029,6 +1051,7 @@ mod test {
             Arc::new(has_z.clone()),
             vec![],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         assert!(SpatialFilter::try_from_expr(&expr_no_args)
             .unwrap_err()
@@ -1041,6 +1064,7 @@ mod test {
             Arc::new(has_z.clone()),
             vec![literal.clone()],
             Arc::new(Field::new("", DataType::Boolean, true)),
+            Arc::new(ConfigOptions::default()),
         ));
         assert!(matches!(
             SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(),

rust/sedona-functions/src/st_setsrid.rs

@@ -232,6 +232,7 @@ mod test {
     use std::rc::Rc;
 
     use arrow_schema::Field;
+    use datafusion_common::config::ConfigOptions;
     use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF};
     use sedona_geometry::{error::SedonaGeometryError, transform::CrsTransform};
     use sedona_schema::{
@@ -374,6 +375,7 @@ mod test {
             arg_fields,
             number_rows: 1,
             return_field,
+            config_options: Arc::new(ConfigOptions::default()),
         };
 
         let value = udf.invoke_with_args(args)?;

rust/sedona-geoparquet/Cargo.toml

@@ -46,6 +46,7 @@ chrono = { workspace = true }
 datafusion = { workspace = true, features = ["parquet"] }
 datafusion-catalog = { workspace = true }
 datafusion-common = { workspace = true }
+datafusion-datasource-parquet = "50.2.0"
 datafusion-execution = { workspace = true }
 datafusion-expr = { workspace = true }
 datafusion-physical-expr = { workspace = true }