diff --git a/Cargo.lock b/Cargo.lock index 120dc29db223b..1c516277c38a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6141,9 +6141,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.59.1" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540683f325ab9ab1a2008bc24588f3e76f63b6a3f52bc47e121122376a063639" +checksum = "21f1cb6d0bcd097a39fc25f7236236be29881fe122e282e4173d6d007a929927" dependencies = [ "heck 0.5.0", "pbjson", @@ -6736,9 +6736,9 @@ checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" [[package]] name = "typify" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7144144e97e987c94758a3017c920a027feac0799df325d6df4fc8f08d02068e" +checksum = "e6d5bcc6f62eb1fa8aa4098f39b29f93dcb914e17158b76c50360911257aa629" dependencies = [ "typify-impl", "typify-macro", @@ -6746,9 +6746,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062879d46aa4c9dfe0d33b035bbaf512da192131645d05deacb7033ec8581a09" +checksum = "a1eb359f7ffa4f9ebe947fa11a1b2da054564502968db5f317b7e37693cb2240" dependencies = [ "heck 0.5.0", "log", @@ -6766,9 +6766,9 @@ dependencies = [ [[package]] name = "typify-macro" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9708a3ceb6660ba3f8d2b8f0567e7d4b8b198e2b94d093b8a6077a751425de9e" +checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" dependencies = [ "proc-macro2", "quote", diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml index 8417bd56852f5..bff9a07a13de2 100644 --- a/datafusion/substrait/Cargo.toml +++ b/datafusion/substrait/Cargo.toml @@ -40,7 +40,7 @@ itertools = { workspace = true } object_store = { workspace = true } pbjson-types = { workspace = true } prost = { workspace = true } -substrait = { version = "0.59", features = ["serde"] } +substrait = { version = "0.62", features = ["serde"] } url = { workspace = true } tokio = { workspace = true, features = ["fs"] } uuid = { version = "1.17.0", features = ["v4"] } diff --git a/datafusion/substrait/src/extensions.rs b/datafusion/substrait/src/extensions.rs index f9a2e0fb82556..0792928982268 100644 --- a/datafusion/substrait/src/extensions.rs +++ b/datafusion/substrait/src/extensions.rs @@ -113,11 +113,15 @@ impl TryFrom<&Vec> for Extensions { } impl From for Vec { + // Silence deprecation warnings for `extension_uri_reference` during the uri -> urn migration + // See: https://github.com/substrait-io/substrait/issues/856 + #[allow(deprecated)] fn from(val: Extensions) -> Vec { let mut extensions = vec![]; for (f_anchor, f_name) in val.functions { let function_extension = ExtensionFunction { extension_uri_reference: u32::MAX, + extension_urn_reference: u32::MAX, function_anchor: f_anchor, name: f_name, }; @@ -130,6 +134,7 @@ impl From for Vec { for (t_anchor, t_name) in val.types { let type_extension = ExtensionType { extension_uri_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545 + extension_urn_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545 type_anchor: t_anchor, name: t_name, }; @@ -142,6 +147,7 @@ impl From for Vec { for (tv_anchor, tv_name) in val.type_variations { let type_variation_extension = ExtensionTypeVariation { extension_uri_reference: u32::MAX, // We don't register proper extension URIs yet + extension_urn_reference: u32::MAX, // We don't register proper extension URIs yet type_variation_anchor: tv_anchor, name: tv_name, }; diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs index d37694ccea05c..f4e43fd586773 100644 --- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs +++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs @@ -60,6 +60,9 @@ use substrait::version; /// /// Substrait also requires the input schema of the expressions to be included in the /// message. The field names of the input schema will be serialized. +// Silence deprecation warnings for `extension_uris` during the uri -> urn migration +// See: https://github.com/substrait-io/substrait/issues/856 +#[allow(deprecated)] pub fn to_substrait_extended_expr( exprs: &[(&Expr, &Field)], schema: &DFSchemaRef, @@ -85,6 +88,7 @@ pub fn to_substrait_extended_expr( advanced_extensions: None, expected_type_urls: vec![], extension_uris: vec![], + extension_urns: vec![], extensions: extensions.into(), version: Some(version::version_with_producer("datafusion")), referred_expr: substrait_exprs, diff --git a/datafusion/substrait/src/logical_plan/producer/plan.rs b/datafusion/substrait/src/logical_plan/producer/plan.rs index 28f6acd0890c7..ad8f45ec3606f 100644 --- a/datafusion/substrait/src/logical_plan/producer/plan.rs +++ b/datafusion/substrait/src/logical_plan/producer/plan.rs @@ -24,6 +24,9 @@ use substrait::proto::{plan_rel, Plan, PlanRel, Rel, RelRoot}; use substrait::version; /// Convert DataFusion LogicalPlan to Substrait Plan +// Silence deprecation warnings for `extension_uris` during the uri -> urn migration +// See: https://github.com/substrait-io/substrait/issues/856 +#[allow(deprecated)] pub fn to_substrait_plan( plan: &LogicalPlan, state: &SessionState, @@ -45,11 +48,13 @@ pub fn to_substrait_plan( Ok(Box::new(Plan { version: Some(version::version_with_producer("datafusion")), extension_uris: vec![], + extension_urns: vec![], extensions: extensions.into(), relations: plan_rels, advanced_extensions: None, expected_type_urls: vec![], parameter_bindings: vec![], + type_aliases: vec![], })) }