Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#793: Added support for Pandas 2 pyarrow dtype columns for emitting data from Python UDFs #357

Merged
merged 13 commits into from
Jun 1, 2023
Merged

#793: Added support for Pandas 2 pyarrow dtype columns for emitting data from Python UDFs #357

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
e1cec26
Update pandas to 2.0.1 in compatible template flavors and replace ass…
tkilias May 4, 2023
b44c274
Add pandas2 with pyarrow support and fix float16 emit bug
tkilias May 16, 2023
0ad7c51
Removed handleEmitNpyFloat16 and debug messages
tkilias May 17, 2023
b7815a5
Add handleEmitPyFloat to also support float pyarrow dtype columns wit…
tkilias May 23, 2023
ff8c9be
Add pandas2 test folder to testconfig
tkilias May 23, 2023
71e8aed
Add support for pyarrow timestamp and decimal dtypes
tkilias May 23, 2023
1309c76
Change astimezone to tz_localize to also handle timestamps without ti…
tkilias May 24, 2023
0845132
Add remaining mixed value/none tests
tkilias May 24, 2023
444e8e2
Fix duplicate error codes
tkilias May 25, 2023
839ca7b
Merge branch 'master' into feature/793_update_to_pandas_2
tkilias May 25, 2023
324f169
Apply review suggestions
tkilias May 31, 2023
ee9cf9d
Merge remote-tracking branch 'origin/master' into feature/793_update_…
tkilias Jun 1, 2023
1770e00
Update to pandas 2.0.2
tkilias Jun 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 180 additions & 68 deletions exaudfclient/base/python/python3/python_ext_dataframe.cc
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion flavors/python-3.7-minimal-EXASOL-6.2.0/flavor_base/testconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
generic_language_tests=python3
test_folders=python3/all
test_folders=python3/all pandas/all
5 changes: 3 additions & 2 deletions ...s/python-3.8-minimal-EXASOL-6.2.0/flavor_base/language_deps/packages/python3_pip_packages
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
pandas|1.3.4
numpy|1.21.3
pandas|2.0.1
numpy|1.24.3
pyarrow|12.0.0
2 changes: 1 addition & 1 deletion flavors/python-3.8-minimal-EXASOL-6.2.0/flavor_base/testconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
generic_language_tests=python3
test_folders=python3/all
test_folders=python3/all pandas/all pandas/pandas2
4 changes: 2 additions & 2 deletions ...rs/template-Exasol-all-python-3.8-conda/flavor_base/language_deps/packages/conda_packages
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
python|3.8.13
numpy|1.22.3
pandas|1.4.2
numpy|1.24.3
pandas|2.0.1
libblas|3.9.0=15_linux64_mkl
mamba|1.3.1
ld_impl_linux-64|2.36.1
Expand Down
2 changes: 1 addition & 1 deletion flavors/template-Exasol-all-python-3.8-conda/flavor_base/testconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
generic_language_tests=python3
test_folders=python3/all
test_folders=python3/all pandas/all pandas/pandas2
4 changes: 2 additions & 2 deletions ...mplate-Exasol-all-python-3.8-cuda-conda/flavor_base/language_deps/packages/conda_packages
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
python|3.8.13
numpy|1.22.3
pandas|1.4.2
numpy|1.24.3
pandas|2.0.1
libblas|3.9.0=15_linux64_mkl
mamba|1.3.1
ld_impl_linux-64|2.36.1
Expand Down
2 changes: 1 addition & 1 deletion flavors/template-Exasol-all-python-3.8-cuda-conda/flavor_base/testconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
generic_language_tests=python3
test_folders=python3/all
test_folders=python3/all pandas/all pandas/pandas2
44 changes: 10 additions & 34 deletions ...ainer/tests/test/python3/all/dataframe.py → ...tainer/tests/test/pandas/all/dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
from datetime import datetime

from exasol_python_test_framework import udf
from exasol_python_test_framework.exatest.testcase import useData
from exasol_python_test_framework.udf.udf_debug import UdfDebugger
from typing import List, Tuple, Union


class PandasDataFrame(udf.TestCase):
def setUp(self):
self.maxDiff=None

self.query('CREATE SCHEMA FN2', ignore_errors=True)
self.query('OPEN SCHEMA FN2', ignore_errors=True)
Expand Down Expand Up @@ -159,7 +163,7 @@ def test_dataframe_scalar_returns(self):

def run(ctx):
df = ctx.get_dataframe()
return np.asscalar(df.iloc[0, 0] + df.iloc[0, 1])
return (df.iloc[0, 0] + df.iloc[0, 1]).item()
/
''' % (self.col_defs_str))
self.query(udf_sql)
Expand Down Expand Up @@ -217,7 +221,7 @@ def test_dataframe_scalar_emits_unique(self):

def run(ctx):
df = ctx.get_dataframe()
ctx.emit(np.asscalar(df.C0))
ctx.emit(df.C0.item())
/
''')
print(udf_sql)
Expand All @@ -236,7 +240,7 @@ def test_dataframe_scalar_emits_all_unique(self):

def run(ctx):
df = ctx.get_dataframe(num_rows="all")
ctx.emit(np.asscalar(df.C0))
ctx.emit(df.C0.item())
/
''')
print(udf_sql)
Expand Down Expand Up @@ -331,7 +335,7 @@ def test_dataframe_set_returns(self):

def run(ctx):
df = ctx.get_dataframe(num_rows="all")
return np.asscalar(df.iloc[:, 0].sum())
return df.iloc[:, 0].sum().item()
/
''' % (self.col_defs_str))
print(udf_sql)
Expand Down Expand Up @@ -477,7 +481,7 @@ def run(ctx):
df = ctx.get_dataframe(num_rows=1)
if df is None:
break
ctx.emit(np.asscalar(df.C0))
ctx.emit(df.C0.item())
/
''')
print(udf_sql)
Expand All @@ -500,7 +504,7 @@ def run(ctx):
if df is None:
break
for i in range(df.shape[0]):
ctx.emit(np.asscalar(df.iloc[i, 0]))
ctx.emit(df.iloc[i, 0].item())
/
''')
print(udf_sql)
Expand Down Expand Up @@ -901,33 +905,6 @@ def run(ctx):
(234,)
], rows)

def test_dataframe_set_emits_double_pyfloat_only_todo(self):
import datetime
udf_sql = udf.fixindent('''
CREATE OR REPLACE PYTHON3 SET SCRIPT foo(sec int) EMITS (ts double) AS

def run(ctx):
import pandas as pd
import numpy as np
import datetime

c1=np.empty(shape=(2),dtype=np.object_)

c1[:]=234.5

df=pd.DataFrame({0:c1})

ctx.emit(df)
/
''')
print(udf_sql)
self.query(udf_sql)
select_sql = 'SELECT foo(1)'
print(select_sql)
#TODO implement support
with self.assertRaisesRegex(Exception, 'F-UDF-CL-SL-PYTHON-1056'):
rows = self.query(select_sql)

def test_dataframe_set_emits_double_npfloat32_only(self):
import datetime
udf_sql = udf.fixindent('''
Expand Down Expand Up @@ -1015,7 +992,6 @@ def run(ctx):
print(select_sql)
rows = self.query(select_sql)


if __name__ == '__main__':
udf.main()

Loading