Skip to content

Commit

Permalink
fix(pyspark): fail when aggregation contains a having filter
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Mar 24, 2023
1 parent 46caf3b commit bd81a9f
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 0 deletions.
7 changes: 7 additions & 0 deletions ibis/backends/pyspark/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,13 @@ def compile_literal(t, op, *, raw=False, **kwargs):
def compile_aggregation(t, op, **kwargs):
src_table = t.translate(op.table, **kwargs)

if op.having:
raise com.UnsupportedOperationError(
"The PySpark backend does not support `having` because the underlying "
"PySpark API does not support it. Use a filter on the aggregation "
"expression instead."
)

if op.predicates:
predicate = functools.reduce(ops.And, op.predicates)
src_table = src_table.filter(t.translate(predicate, **kwargs))
Expand Down
5 changes: 5 additions & 0 deletions ibis/backends/tests/test_generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -1125,6 +1125,11 @@ def test_distinct_on_keep(backend, on, keep):
raises=com.OperationNotDefinedError,
reason="backend doesn't implement ops.WindowFunction",
)
@pytest.mark.notimpl(
["pyspark"],
raises=com.UnsupportedOperationError,
reason="backend doesn't support `having` filters",
)
def test_distinct_on_keep_is_none(backend, on):
from ibis import _

Expand Down

0 comments on commit bd81a9f

Please sign in to comment.