Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions datafusion-examples/examples/custom_file_casts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,18 @@ impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {
// For example, [DataFusion Comet](https://github.com/apache/datafusion-comet) has a [custom cast kernel](https://github.com/apache/datafusion-comet/blob/b4ac876ab420ed403ac7fc8e1b29f42f1f442566/native/spark-expr/src/conversion_funcs/cast.rs#L133-L138).
expr.transform(|expr| {
if let Some(cast) = expr.as_any().downcast_ref::<CastExpr>() {
let input_data_type = cast.expr().data_type(&self.physical_file_schema)?;
let input_data_type =
cast.expr().data_type(&self.physical_file_schema)?;
let output_data_type = cast.data_type(&self.physical_file_schema)?;
if !cast.is_bigger_cast(&input_data_type) {
return not_impl_err!("Unsupported CAST from {input_data_type:?} to {output_data_type:?}")
return not_impl_err!(
"Unsupported CAST from {input_data_type} to {output_data_type}"
);
}
}
Ok(Transformed::no(expr))
}).data()
})
.data()
}

fn with_partition_values(
Expand Down
5 changes: 2 additions & 3 deletions datafusion/catalog/src/information_schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ fn get_udwf_args_and_return_types(

#[inline]
fn remove_native_type_prefix(native_type: NativeType) -> String {
format!("{native_type:?}")
format!("{native_type}")
}

#[async_trait]
Expand Down Expand Up @@ -827,8 +827,7 @@ impl InformationSchemaColumnsBuilder {
self.is_nullables.append_value(nullable_str);

// "System supplied type" --> Use debug format of the datatype
self.data_types
.append_value(format!("{:?}", field.data_type()));
self.data_types.append_value(field.data_type().to_string());

// "If data_type identifies a character or bit string type, the
// declared maximum length; null for all other data types or
Expand Down
10 changes: 5 additions & 5 deletions datafusion/common/src/dfschema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -669,8 +669,8 @@ impl DFSchema {
))
{
_plan_err!(
"Schema mismatch: Expected field '{}' with type {:?}, \
but got '{}' with type {:?}.",
"Schema mismatch: Expected field '{}' with type {}, \
but got '{}' with type {}.",
f1.name(),
f1.data_type(),
f2.name(),
Expand Down Expand Up @@ -1063,7 +1063,7 @@ fn format_simple_data_type(data_type: &DataType) -> String {
format!("decimal256({precision}, {scale})")
}
DataType::Null => "null".to_string(),
_ => format!("{data_type:?}").to_lowercase(),
_ => format!("{data_type}").to_lowercase(),
}
}

Expand Down Expand Up @@ -1308,8 +1308,8 @@ impl SchemaExt for Schema {
.try_for_each(|(f1, f2)| {
if f1.name() != f2.name() || (!DFSchema::datatype_is_logically_equal(f1.data_type(), f2.data_type()) && !can_cast_types(f2.data_type(), f1.data_type())) {
_plan_err!(
"Inserting query schema mismatch: Expected table field '{}' with type {:?}, \
but got '{}' with type {:?}.",
"Inserting query schema mismatch: Expected table field '{}' with type {}, \
but got '{}' with type {}.",
f1.name(),
f1.data_type(),
f2.name(),
Expand Down
12 changes: 6 additions & 6 deletions datafusion/common/src/nested_struct.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ fn cast_struct_column(
} else {
// Return error if source is not a struct type
Err(DataFusionError::Plan(format!(
"Cannot cast column of type {:?} to struct type. Source must be a struct to cast to struct.",
"Cannot cast column of type {} to struct type. Source must be a struct to cast to struct.",
source_col.data_type()
)))
}
Expand All @@ -98,7 +98,7 @@ fn cast_struct_column(
/// The struct casting logic requires that the source column must already be a struct type.
/// This makes the function useful for:
/// - Schema evolution scenarios where struct layouts change over time
/// - Data migration between different struct schemas
/// - Data migration between different struct schemas
/// - Type-safe data processing pipelines that maintain struct type integrity
///
/// # Arguments
Expand All @@ -124,15 +124,15 @@ pub fn cast_column(source_col: &ArrayRef, target_field: &Field) -> Result<ArrayR
/// Validates compatibility between source and target struct fields for casting operations.
///
/// This function implements comprehensive struct compatibility checking by examining:
/// - Field name matching between source and target structs
/// - Field name matching between source and target structs
/// - Type castability for each matching field (including recursive struct validation)
/// - Proper handling of missing fields (target fields not in source are allowed - filled with nulls)
/// - Proper handling of extra fields (source fields not in target are allowed - ignored)
///
/// # Compatibility Rules
/// - **Field Matching**: Fields are matched by name (case-sensitive)
/// - **Missing Target Fields**: Allowed - will be filled with null values during casting
/// - **Extra Source Fields**: Allowed - will be ignored during casting
/// - **Extra Source Fields**: Allowed - will be ignored during casting
/// - **Type Compatibility**: Each matching field must be castable using Arrow's type system
/// - **Nested Structs**: Recursively validates nested struct compatibility
///
Expand All @@ -147,7 +147,7 @@ pub fn cast_column(source_col: &ArrayRef, target_field: &Field) -> Result<ArrayR
/// # Examples
/// ```text
/// // Compatible: source has extra field, target has missing field
/// // Source: {a: i32, b: string, c: f64}
/// // Source: {a: i32, b: string, c: f64}
/// // Target: {a: i64, d: bool}
/// // Result: Ok(true) - 'a' can cast i32->i64, 'b','c' ignored, 'd' filled with nulls
///
Expand Down Expand Up @@ -180,7 +180,7 @@ pub fn validate_struct_compatibility(
target_field.data_type(),
) {
return _plan_err!(
"Cannot cast struct field '{}' from type {:?} to type {:?}",
"Cannot cast struct field '{}' from type {} to type {}",
target_field.name(),
source_field.data_type(),
target_field.data_type()
Expand Down
2 changes: 1 addition & 1 deletion datafusion/common/src/param_value.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ impl ParamValues {
for (i, (param_type, value)) in iter.enumerate() {
if *param_type != value.data_type() {
return _plan_err!(
"Expected parameter of type {:?}, got {:?} at index {}",
"Expected parameter of type {}, got {:?} at index {}",
param_type,
value.data_type(),
i
Expand Down
48 changes: 21 additions & 27 deletions datafusion/common/src/scalar/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1137,7 +1137,7 @@ impl ScalarValue {
DataType::Null => ScalarValue::Null,
_ => {
return _not_impl_err!(
"Can't create a null scalar from data_type \"{data_type:?}\""
"Can't create a null scalar from data_type \"{data_type}\""
);
}
})
Expand Down Expand Up @@ -1193,7 +1193,7 @@ impl ScalarValue {
match datatype {
DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::PI)),
DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::PI)),
_ => _internal_err!("PI is not supported for data type: {:?}", datatype),
_ => _internal_err!("PI is not supported for data type: {}", datatype),
}
}

Expand All @@ -1203,7 +1203,7 @@ impl ScalarValue {
DataType::Float32 => Ok(ScalarValue::from(consts::PI_UPPER_F32)),
DataType::Float64 => Ok(ScalarValue::from(consts::PI_UPPER_F64)),
_ => {
_internal_err!("PI_UPPER is not supported for data type: {:?}", datatype)
_internal_err!("PI_UPPER is not supported for data type: {}", datatype)
}
}
}
Expand All @@ -1214,7 +1214,7 @@ impl ScalarValue {
DataType::Float32 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F32)),
DataType::Float64 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F64)),
_ => {
_internal_err!("-PI_LOWER is not supported for data type: {:?}", datatype)
_internal_err!("-PI_LOWER is not supported for data type: {}", datatype)
}
}
}
Expand All @@ -1225,10 +1225,7 @@ impl ScalarValue {
DataType::Float32 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F32)),
DataType::Float64 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F64)),
_ => {
_internal_err!(
"PI_UPPER/2 is not supported for data type: {:?}",
datatype
)
_internal_err!("PI_UPPER/2 is not supported for data type: {}", datatype)
}
}
}
Expand All @@ -1243,10 +1240,7 @@ impl ScalarValue {
Ok(ScalarValue::from(consts::NEGATIVE_FRAC_PI_2_LOWER_F64))
}
_ => {
_internal_err!(
"-PI/2_LOWER is not supported for data type: {:?}",
datatype
)
_internal_err!("-PI/2_LOWER is not supported for data type: {}", datatype)
}
}
}
Expand All @@ -1256,7 +1250,7 @@ impl ScalarValue {
match datatype {
DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::PI)),
DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::PI)),
_ => _internal_err!("-PI is not supported for data type: {:?}", datatype),
_ => _internal_err!("-PI is not supported for data type: {}", datatype),
}
}

Expand All @@ -1265,7 +1259,7 @@ impl ScalarValue {
match datatype {
DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::FRAC_PI_2)),
DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::FRAC_PI_2)),
_ => _internal_err!("PI/2 is not supported for data type: {:?}", datatype),
_ => _internal_err!("PI/2 is not supported for data type: {}", datatype),
}
}

Expand All @@ -1274,7 +1268,7 @@ impl ScalarValue {
match datatype {
DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::FRAC_PI_2)),
DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::FRAC_PI_2)),
_ => _internal_err!("-PI/2 is not supported for data type: {:?}", datatype),
_ => _internal_err!("-PI/2 is not supported for data type: {}", datatype),
}
}

Expand All @@ -1284,7 +1278,7 @@ impl ScalarValue {
DataType::Float32 => Ok(ScalarValue::from(f32::INFINITY)),
DataType::Float64 => Ok(ScalarValue::from(f64::INFINITY)),
_ => {
_internal_err!("Infinity is not supported for data type: {:?}", datatype)
_internal_err!("Infinity is not supported for data type: {}", datatype)
}
}
}
Expand All @@ -1296,7 +1290,7 @@ impl ScalarValue {
DataType::Float64 => Ok(ScalarValue::from(f64::NEG_INFINITY)),
_ => {
_internal_err!(
"Negative Infinity is not supported for data type: {:?}",
"Negative Infinity is not supported for data type: {}",
datatype
)
}
Expand Down Expand Up @@ -1369,7 +1363,7 @@ impl ScalarValue {
DataType::Date64 => ScalarValue::Date64(Some(0)),
_ => {
return _not_impl_err!(
"Can't create a zero scalar from data_type \"{datatype:?}\""
"Can't create a zero scalar from data_type \"{datatype}\""
);
}
})
Expand Down Expand Up @@ -1507,7 +1501,7 @@ impl ScalarValue {
// Unsupported types for now
_ => {
_not_impl_err!(
"Default value for data_type \"{datatype:?}\" is not implemented yet"
"Default value for data_type \"{datatype}\" is not implemented yet"
)
}
}
Expand Down Expand Up @@ -1557,7 +1551,7 @@ impl ScalarValue {
}
_ => {
return _not_impl_err!(
"Can't create an one scalar from data_type \"{datatype:?}\""
"Can't create an one scalar from data_type \"{datatype}\""
);
}
})
Expand Down Expand Up @@ -1603,7 +1597,7 @@ impl ScalarValue {
}
_ => {
return _not_impl_err!(
"Can't create a negative one scalar from data_type \"{datatype:?}\""
"Can't create a negative one scalar from data_type \"{datatype}\""
);
}
})
Expand Down Expand Up @@ -1656,7 +1650,7 @@ impl ScalarValue {
}
_ => {
return _not_impl_err!(
"Can't create a ten scalar from data_type \"{datatype:?}\""
"Can't create a ten scalar from data_type \"{datatype}\""
);
}
})
Expand Down Expand Up @@ -2364,7 +2358,7 @@ impl ScalarValue {
DataType::UInt16 => dict_from_values::<UInt16Type>(values)?,
DataType::UInt32 => dict_from_values::<UInt32Type>(values)?,
DataType::UInt64 => dict_from_values::<UInt64Type>(values)?,
_ => unreachable!("Invalid dictionary keys type: {:?}", key_type),
_ => unreachable!("Invalid dictionary keys type: {}", key_type),
}
}
DataType::FixedSizeBinary(size) => {
Expand All @@ -2375,7 +2369,7 @@ impl ScalarValue {
} else {
_exec_err!(
"Inconsistent types in ScalarValue::iter_to_array. \
Expected {data_type:?}, got {sv:?}"
Expected {data_type}, got {sv:?}"
)
}
})
Expand Down Expand Up @@ -2937,7 +2931,7 @@ impl ScalarValue {
DataType::UInt16 => dict_from_scalar::<UInt16Type>(v, size)?,
DataType::UInt32 => dict_from_scalar::<UInt32Type>(v, size)?,
DataType::UInt64 => dict_from_scalar::<UInt64Type>(v, size)?,
_ => unreachable!("Invalid dictionary keys type: {:?}", key_type),
_ => unreachable!("Invalid dictionary keys type: {}", key_type),
}
}
ScalarValue::Null => get_or_create_cached_null_array(size),
Expand Down Expand Up @@ -3197,7 +3191,7 @@ impl ScalarValue {
DataType::UInt16 => get_dict_value::<UInt16Type>(array, index)?,
DataType::UInt32 => get_dict_value::<UInt32Type>(array, index)?,
DataType::UInt64 => get_dict_value::<UInt64Type>(array, index)?,
_ => unreachable!("Invalid dictionary keys type: {:?}", key_type),
_ => unreachable!("Invalid dictionary keys type: {}", key_type),
};
// look up the index in the values dictionary
let value = match values_index {
Expand Down Expand Up @@ -3571,7 +3565,7 @@ impl ScalarValue {
DataType::UInt16 => get_dict_value::<UInt16Type>(array, index)?,
DataType::UInt32 => get_dict_value::<UInt32Type>(array, index)?,
DataType::UInt64 => get_dict_value::<UInt64Type>(array, index)?,
_ => unreachable!("Invalid dictionary keys type: {:?}", key_type),
_ => unreachable!("Invalid dictionary keys type: {}", key_type),
};
// was the value in the array non null?
match values_index {
Expand Down
8 changes: 4 additions & 4 deletions datafusion/common/src/types/native.rs
Original file line number Diff line number Diff line change
Expand Up @@ -352,10 +352,10 @@ impl LogicalType for NativeType {
}
_ => {
return _internal_err!(
"Unavailable default cast for native type {:?} from physical type {:?}",
self,
origin
)
"Unavailable default cast for native type {} from physical type {}",
self,
origin
)
}
})
}
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/src/physical_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2111,7 +2111,7 @@ impl DefaultPhysicalPlanner {

// "System supplied type" --> Use debug format of the datatype
let data_type = field.data_type();
data_types.append_value(format!("{data_type:?}"));
data_types.append_value(format!("{data_type}"));

// "YES if the column is possibly nullable, NO if it is known not nullable. "
let nullable_str = if field.is_nullable() { "YES" } else { "NO" };
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/tests/parquet/schema_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ impl SchemaMapper for CustomSchemaMapper {
let default_value = match field.data_type() {
DataType::Int64 => ScalarValue::Int64(Some(0)),
DataType::Utf8 => ScalarValue::Utf8(Some("a".to_string())),
_ => unimplemented!("Unsupported data type: {:?}", field.data_type()),
_ => unimplemented!("Unsupported data type: {}", field.data_type()),
};
output_columns
.push(default_value.to_array_of_size(batch.num_rows()).unwrap());
Expand Down Expand Up @@ -199,7 +199,7 @@ impl PhysicalExprAdapter for CustomPhysicalExprAdapter {
DataType::Int64 => ScalarValue::Int64(Some(1)),
DataType::Utf8 => ScalarValue::Utf8(Some("b".to_string())),
_ => unimplemented!(
"Unsupported data type: {:?}",
"Unsupported data type: {}",
field.data_type()
),
};
Expand Down
Loading
Loading