Skip to content

[DF] WIP: Upgrade to DataFusion rev 8ea59a #779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
charlesbluca merged 9 commits into from
Sep 21, 2022
Merged

[DF] WIP: Upgrade to DataFusion rev 8ea59a #779

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9c3ef9d
save progress
andygrove Sep 20, 2022
4ff072a
partial fix
andygrove Sep 20, 2022
a977108
Add accessor functions for time64 and timestamp values
ayushdg Sep 20, 2022
f2598a7
Add handling for time,timestamp literals
ayushdg Sep 20, 2022
eb89905
Merge branch 'datafusion-sql-planner' of github.com:dask-contrib/dask…
ayushdg Sep 21, 2022
b5b7acd
Re-add log env_log removed during merge
ayushdg Sep 21, 2022
fdb62bc
Remove debug/commented code
ayushdg Sep 21, 2022
36fbd25
Un-xfail query 32
charlesbluca Sep 21, 2022
665e538
Implement support for scalar Decimal128
charlesbluca Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 29 additions & 17 deletions dask_planner/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions dask_planner/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ edition = "2021"
rust-version = "1.62"

[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
datafusion-common = "12.0.0"
datafusion-expr = "12.0.0"
datafusion-optimizer = "12.0.0"
datafusion-sql = "12.0.0"
datafusion-common = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
datafusion-expr = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
datafusion-sql = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
env_logger = "0.9"
log = "^0.4"
mimalloc = { version = "*", default-features = false }
Expand Down
118 changes: 86 additions & 32 deletions dask_planner/src/expression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,152 +529,206 @@ impl PyExpr {
/// TODO: I can't express how much I dislike explicity listing all of these methods out
/// but PyO3 makes it necessary since its annotations cannot be used in trait impl blocks
#[pyo3(name = "getFloat32Value")]
pub fn float_32_value(&mut self) -> PyResult<f32> {
pub fn float_32_value(&mut self) -> PyResult<Option<f32>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Float32(iv) => Ok(iv.unwrap()),
ScalarValue::Float32(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getFloat64Value")]
pub fn float_64_value(&mut self) -> PyResult<f64> {
pub fn float_64_value(&mut self) -> PyResult<Option<f64>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Float64(iv) => Ok(iv.unwrap()),
ScalarValue::Float64(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getInt8Value")]
pub fn int_8_value(&mut self) -> PyResult<i8> {
pub fn int_8_value(&mut self) -> PyResult<Option<i8>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Int8(iv) => Ok(iv.unwrap()),
ScalarValue::Int8(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getInt16Value")]
pub fn int_16_value(&mut self) -> PyResult<i16> {
pub fn int_16_value(&mut self) -> PyResult<Option<i16>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Int16(iv) => Ok(iv.unwrap()),
ScalarValue::Int16(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getInt32Value")]
pub fn int_32_value(&mut self) -> PyResult<i32> {
pub fn int_32_value(&mut self) -> PyResult<Option<i32>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Int32(iv) => Ok(iv.unwrap()),
ScalarValue::Int32(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getInt64Value")]
pub fn int_64_value(&mut self) -> PyResult<i64> {
pub fn int_64_value(&mut self) -> PyResult<Option<i64>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Int64(iv) => Ok(iv.unwrap()),
ScalarValue::Int64(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getUInt8Value")]
pub fn uint_8_value(&mut self) -> PyResult<u8> {
pub fn uint_8_value(&mut self) -> PyResult<Option<u8>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::UInt8(iv) => Ok(iv.unwrap()),
ScalarValue::UInt8(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getUInt16Value")]
pub fn uint_16_value(&mut self) -> PyResult<u16> {
pub fn uint_16_value(&mut self) -> PyResult<Option<u16>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::UInt16(iv) => Ok(iv.unwrap()),
ScalarValue::UInt16(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getUInt32Value")]
pub fn uint_32_value(&mut self) -> PyResult<u32> {
pub fn uint_32_value(&mut self) -> PyResult<Option<u32>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::UInt32(iv) => Ok(iv.unwrap()),
ScalarValue::UInt32(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getUInt64Value")]
pub fn uint_64_value(&mut self) -> PyResult<u64> {
pub fn uint_64_value(&mut self) -> PyResult<Option<u64>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::UInt64(iv) => Ok(iv.unwrap()),
ScalarValue::UInt64(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getDate32Value")]
pub fn date_32_value(&mut self) -> PyResult<Option<i32>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Date32(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getDate64Value")]
pub fn date_64_value(&mut self) -> PyResult<Option<i64>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Date64(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getTime64Value")]
pub fn time_64_value(&mut self) -> PyResult<Option<i64>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Time64(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getTimestampValue")]
pub fn timestamp_value(&mut self) -> PyResult<(Option<i64>, Option<String>)> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::TimestampNanosecond(iv, tz)
| ScalarValue::TimestampMicrosecond(iv, tz)
| ScalarValue::TimestampMillisecond(iv, tz)
| ScalarValue::TimestampSecond(iv, tz) => {
if let Some(iv) = iv {
println!("{}", iv);
}
if let Some(tz) = tz {
println!("{}", tz);
}
Comment thread
ayushdg marked this conversation as resolved.
Outdated
Ok((*iv, tz.clone()))
}
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getBoolValue")]
pub fn bool_value(&mut self) -> PyResult<bool> {
pub fn bool_value(&mut self) -> PyResult<Option<bool>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Boolean(Some(iv)) => Ok(*iv),
ScalarValue::Boolean(iv) => Ok(*iv),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getStringValue")]
pub fn string_value(&mut self) -> PyResult<String> {
pub fn string_value(&mut self) -> PyResult<Option<String>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::Utf8(iv) => Ok(iv.clone().unwrap()),
ScalarValue::Utf8(iv) => Ok(iv.clone()),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

#[pyo3(name = "getIntervalDayTimeValue")]
pub fn interval_day_time_value(&mut self) -> (i32, i32) {
pub fn interval_day_time_value(&mut self) -> PyResult<Option<(i32, i32)>> {
match &self.expr {
Expr::Literal(scalar_value) => match scalar_value {
ScalarValue::IntervalDayTime(iv) => {
let interval = iv.unwrap() as u64;
ScalarValue::IntervalDayTime(Some(iv)) => {
let interval = *iv as u64;
let days = (interval >> 32) as i32;
let ms = interval as i32;
(days, ms)
}
_ => {
panic!("getValue<T>() - Unexpected value")
Ok(Some((days, ms)))
}
ScalarValue::IntervalDayTime(None) => Ok(None),
_ => Err(py_type_err("getValue<T>() - Unexpected value")),
},
_ => panic!("getValue<T>() - Non literal value encountered"),
_ => Err(py_type_err("getValue<T>() - Non literal value encountered")),
}
}

Expand Down
Loading