diff --git a/arrow-pyarrow-integration-testing/Cargo.toml b/arrow-pyarrow-integration-testing/Cargo.toml index c757f6739373..7eecf8810f7b 100644 --- a/arrow-pyarrow-integration-testing/Cargo.toml +++ b/arrow-pyarrow-integration-testing/Cargo.toml @@ -34,4 +34,4 @@ crate-type = ["cdylib"] [dependencies] arrow = { path = "../arrow", features = ["pyarrow"] } -pyo3 = { version = "0.25.1", features = ["extension-module"] } +pyo3 = { version = "0.26.0", features = ["extension-module"] } diff --git a/arrow-pyarrow-integration-testing/src/lib.rs b/arrow-pyarrow-integration-testing/src/lib.rs index d4908fff0897..86c17ab79caa 100644 --- a/arrow-pyarrow-integration-testing/src/lib.rs +++ b/arrow-pyarrow-integration-testing/src/lib.rs @@ -41,7 +41,8 @@ fn to_py_err(err: ArrowError) -> PyErr { /// Returns `array + array` of an int64 array. #[pyfunction] -fn double(array: &Bound, py: Python) -> PyResult { +fn double<'py>(array: &Bound<'py, PyAny>) -> PyResult> { + let py = array.py(); // import let array = make_array(ArrayData::from_pyarrow_bound(array)?); @@ -61,13 +62,13 @@ fn double(array: &Bound, py: Python) -> PyResult { /// calls a lambda function that receives and returns an array /// whose result must be the array multiplied by two #[pyfunction] -fn double_py(lambda: &Bound, py: Python) -> PyResult { +fn double_py(lambda: &Bound) -> PyResult { // create let array = Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])); let expected = Arc::new(Int64Array::from(vec![Some(2), None, Some(6)])) as ArrayRef; // to py - let pyarray = array.to_data().to_pyarrow(py)?; + let pyarray = array.to_data().to_pyarrow(lambda.py())?; let pyarray = lambda.call1((pyarray,))?; let array = make_array(ArrayData::from_pyarrow_bound(&pyarray)?); @@ -75,7 +76,10 @@ fn double_py(lambda: &Bound, py: Python) -> PyResult { } #[pyfunction] -fn make_empty_array(datatype: PyArrowType, py: Python) -> PyResult { +fn make_empty_array<'py>( + datatype: PyArrowType, + py: Python<'py>, +) -> PyResult> { let array = new_empty_array(&datatype.0); array.to_data().to_pyarrow(py) @@ -95,7 +99,7 @@ fn substring(array: PyArrowType, start: i64) -> PyResult, py: Python) -> PyResult { +fn concatenate<'py>(array: PyArrowType, py: Python<'py>) -> PyResult> { let array = make_array(array.0); // concat diff --git a/arrow-pyarrow-testing/Cargo.toml b/arrow-pyarrow-testing/Cargo.toml index 8bbf364f2e08..e5ba0f49f035 100644 --- a/arrow-pyarrow-testing/Cargo.toml +++ b/arrow-pyarrow-testing/Cargo.toml @@ -48,4 +48,4 @@ publish = false # Note no dependency on arrow, to ensure arrow-pyarrow can be used by itself arrow-array = { path = "../arrow-array" } arrow-pyarrow = { path = "../arrow-pyarrow" } -pyo3 = { version = "0.25", default-features = false } +pyo3 = { version = "0.26.0", default-features = false } diff --git a/arrow-pyarrow-testing/tests/pyarrow.rs b/arrow-pyarrow-testing/tests/pyarrow.rs index 3d3c30cf210a..4ca661b104d2 100644 --- a/arrow-pyarrow-testing/tests/pyarrow.rs +++ b/arrow-pyarrow-testing/tests/pyarrow.rs @@ -47,7 +47,7 @@ use std::sync::Arc; #[test] fn test_to_pyarrow() { - pyo3::prepare_freethreaded_python(); + Python::initialize(); let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); let b: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); @@ -56,11 +56,11 @@ fn test_to_pyarrow() { let input = RecordBatch::try_from_iter(vec![("a", a), ("b", b), ("c", c)]).unwrap(); println!("input: {input:?}"); - let res = Python::with_gil(|py| { + let res = Python::attach(|py| { let py_input = input.to_pyarrow(py)?; - let records = RecordBatch::from_pyarrow_bound(py_input.bind(py))?; + let records = RecordBatch::from_pyarrow_bound(&py_input)?; let py_records = records.to_pyarrow(py)?; - RecordBatch::from_pyarrow_bound(py_records.bind(py)) + RecordBatch::from_pyarrow_bound(&py_records) }) .unwrap(); @@ -69,7 +69,7 @@ fn test_to_pyarrow() { #[test] fn test_to_pyarrow_byte_view() { - pyo3::prepare_freethreaded_python(); + Python::initialize(); for num_variadic_buffers in 0..=2 { let string_view: ArrayRef = Arc::new(string_view_column(num_variadic_buffers)); @@ -82,11 +82,11 @@ fn test_to_pyarrow_byte_view() { .unwrap(); println!("input: {input:?}"); - let res = Python::with_gil(|py| { + let res = Python::attach(|py| { let py_input = input.to_pyarrow(py)?; - let records = RecordBatch::from_pyarrow_bound(py_input.bind(py))?; + let records = RecordBatch::from_pyarrow_bound(&py_input)?; let py_records = records.to_pyarrow(py)?; - RecordBatch::from_pyarrow_bound(py_records.bind(py)) + RecordBatch::from_pyarrow_bound(&py_records) }) .unwrap(); diff --git a/arrow-pyarrow/Cargo.toml b/arrow-pyarrow/Cargo.toml index 9eeab3796617..9cfa235324f1 100644 --- a/arrow-pyarrow/Cargo.toml +++ b/arrow-pyarrow/Cargo.toml @@ -39,4 +39,4 @@ all-features = true arrow-array = { workspace = true, features = ["ffi"] } arrow-data = { workspace = true } arrow-schema = { workspace = true } -pyo3 = { version = "0.25.1", default-features = false } +pyo3 = { version = "0.26.0", default-features = false } diff --git a/arrow-pyarrow/src/lib.rs b/arrow-pyarrow/src/lib.rs index c958da9d1c92..a238b4abbb07 100644 --- a/arrow-pyarrow/src/lib.rs +++ b/arrow-pyarrow/src/lib.rs @@ -95,17 +95,17 @@ pub trait FromPyArrow: Sized { /// Create a new PyArrow object from a arrow-rs type. pub trait ToPyArrow { /// Convert the implemented type into a Python object without consuming it. - fn to_pyarrow(&self, py: Python) -> PyResult; + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult>; } /// Convert an arrow-rs type into a PyArrow object. pub trait IntoPyArrow { /// Convert the implemented type into a Python object while consuming it. - fn into_pyarrow(self, py: Python) -> PyResult; + fn into_pyarrow<'py>(self, py: Python<'py>) -> PyResult>; } impl IntoPyArrow for T { - fn into_pyarrow(self, py: Python) -> PyResult { + fn into_pyarrow<'py>(self, py: Python<'py>) -> PyResult> { self.to_pyarrow(py) } } @@ -172,7 +172,7 @@ impl FromPyArrow for DataType { } impl ToPyArrow for DataType { - fn to_pyarrow(&self, py: Python) -> PyResult { + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let c_schema = FFI_ArrowSchema::try_from(self).map_err(to_py_err)?; let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; let module = py.import("pyarrow")?; @@ -208,7 +208,7 @@ impl FromPyArrow for Field { } impl ToPyArrow for Field { - fn to_pyarrow(&self, py: Python) -> PyResult { + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let c_schema = FFI_ArrowSchema::try_from(self).map_err(to_py_err)?; let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; let module = py.import("pyarrow")?; @@ -244,7 +244,7 @@ impl FromPyArrow for Schema { } impl ToPyArrow for Schema { - fn to_pyarrow(&self, py: Python) -> PyResult { + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let c_schema = FFI_ArrowSchema::try_from(self).map_err(to_py_err)?; let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; let module = py.import("pyarrow")?; @@ -303,7 +303,7 @@ impl FromPyArrow for ArrayData { } impl ToPyArrow for ArrayData { - fn to_pyarrow(&self, py: Python) -> PyResult { + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let array = FFI_ArrowArray::new(self); let schema = FFI_ArrowSchema::try_from(self.data_type()).map_err(to_py_err)?; @@ -316,7 +316,7 @@ impl ToPyArrow for ArrayData { addr_of!(schema) as Py_uintptr_t, ), )?; - Ok(array.unbind()) + Ok(array) } } @@ -328,12 +328,12 @@ impl FromPyArrow for Vec { } impl ToPyArrow for Vec { - fn to_pyarrow(&self, py: Python) -> PyResult { + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let values = self .iter() .map(|v| v.to_pyarrow(py)) .collect::>>()?; - Ok(PyList::new(py, values)?.unbind().into()) + Ok(PyList::new(py, values)?.into_any()) } } @@ -412,12 +412,12 @@ impl FromPyArrow for RecordBatch { } impl ToPyArrow for RecordBatch { - fn to_pyarrow(&self, py: Python) -> PyResult { + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { // Workaround apache/arrow#37669 by returning RecordBatchIterator let reader = RecordBatchIterator::new(vec![Ok(self.clone())], self.schema()); let reader: Box = Box::new(reader); let py_reader = reader.into_pyarrow(py)?; - py_reader.call_method0(py, "read_next_batch") + py_reader.call_method0("read_next_batch") } } @@ -463,7 +463,7 @@ impl FromPyArrow for ArrowArrayStreamReader { impl IntoPyArrow for Box { // We can't implement `ToPyArrow` for `T: RecordBatchReader + Send` because // there is already a blanket implementation for `T: ToPyArrow`. - fn into_pyarrow(self, py: Python) -> PyResult { + fn into_pyarrow<'py>(self, py: Python<'py>) -> PyResult> { let mut stream = FFI_ArrowArrayStream::new(self); let stream_ptr = (&mut stream) as *mut FFI_ArrowArrayStream; @@ -472,13 +472,13 @@ impl IntoPyArrow for Box { let args = PyTuple::new(py, [stream_ptr as Py_uintptr_t])?; let reader = class.call_method1("_import_from_c", args)?; - Ok(PyObject::from(reader)) + Ok(reader) } } /// Convert a [`ArrowArrayStreamReader`] into a `pyarrow.RecordBatchReader`. impl IntoPyArrow for ArrowArrayStreamReader { - fn into_pyarrow(self, py: Python) -> PyResult { + fn into_pyarrow<'py>(self, py: Python<'py>) -> PyResult> { let boxed: Box = Box::new(self); boxed.into_pyarrow(py) } @@ -506,10 +506,7 @@ impl<'py, T: IntoPyArrow> IntoPyObject<'py> for PyArrowType { type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - match self.0.into_pyarrow(py) { - Ok(obj) => Result::Ok(obj.into_bound(py)), - Err(err) => Result::Err(err), - } + self.0.into_pyarrow(py) } }