Skip to content

Add more tests showing coercing behavior with literals #14270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alamb merged 4 commits into from
Jan 26, 2025
Merged

Add more tests showing coercing behavior with literals #14270

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
befe6de
Add more tests showing coercing behavior with literals
alamb Jan 24, 2025
d487402
remove unecessary change
alamb Jan 24, 2025
4b32876
Merge remote-tracking branch 'apache/main' into alamb/more_casting_tests
alamb Jan 25, 2025
62e5526
Add tests for comparisons
alamb Jan 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 250 additions & 0 deletions datafusion/sqllogictest/test_files/operator.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ SELECT
arrow_cast(1.25, 'Decimal128(5, 2)') as decimal
;

############### Addition ###############

# Plus with the same operand type, expect the same output type
# except for decimal which is promoted to the highest precision
query TTTTTTTTTTT
Expand All @@ -52,6 +54,43 @@ from numeric_types;
----
Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 Float32 Float64 Decimal128(6, 2)

# Plus with literal integer
query TTTTTTTTTTT
select
arrow_typeof(int8 + 2),
arrow_typeof(int16 + 2),
arrow_typeof(int32 + 2),
arrow_typeof(int64 + 2),
arrow_typeof(uint8 + 2),
arrow_typeof(uint16 + 2),
arrow_typeof(uint32 + 2),
arrow_typeof(uint64 + 2),
arrow_typeof(float32 + 2),
arrow_typeof(float64 + 2),
arrow_typeof(decimal + 2)
from numeric_types;
----
Int64 Int64 Int64 Int64 Int64 Int64 Int64 Int64 Float32 Float64 Decimal128(23, 2)
Copy link
Contributor Author

@alamb alamb Jan 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This shows that adding a literal integer (2) does not upcast to Decimal for integer types (which would be terrible for performance)


# Plus with literal decimal
query TTTTTTTTTTT
select
arrow_typeof(int8 + 2.0),
arrow_typeof(int16 + 2.0),
arrow_typeof(int32 + 2.0),
arrow_typeof(int64 + 2.0),
arrow_typeof(uint8 + 2.0),
arrow_typeof(uint16 + 2.0),
arrow_typeof(uint32 + 2.0),
arrow_typeof(uint64 + 2.0),
arrow_typeof(float32 + 2.0),
arrow_typeof(float64 + 2.0),
arrow_typeof(decimal + 2.0)
from numeric_types;
----
Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64

############### Subtraction ###############

# Minus with the same operand type, expect the same output type
# except for decimal which is promoted to the highest precision
Expand All @@ -72,6 +111,44 @@ from numeric_types;
----
Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 Float32 Float64 Decimal128(6, 2)

# Minus with literal integer
query TTTTTTTTTTT
select
arrow_typeof(int8 - 2),
arrow_typeof(int16 - 2),
arrow_typeof(int32 - 2),
arrow_typeof(int64 - 2),
arrow_typeof(uint8 - 2),
arrow_typeof(uint16 - 2),
arrow_typeof(uint32 - 2),
arrow_typeof(uint64 - 2),
arrow_typeof(float32 - 2),
arrow_typeof(float64 - 2),
arrow_typeof(decimal - 2)
from numeric_types;
----
Int64 Int64 Int64 Int64 Int64 Int64 Int64 Int64 Float32 Float64 Decimal128(23, 2)

# Minus with literal decimal
query TTTTTTTTTTT
select
arrow_typeof(int8 - 2.0),
arrow_typeof(int16 - 2.0),
arrow_typeof(int32 - 2.0),
arrow_typeof(int64 - 2.0),
arrow_typeof(uint8 - 2.0),
arrow_typeof(uint16 - 2.0),
arrow_typeof(uint32 - 2.0),
arrow_typeof(uint64 - 2.0),
arrow_typeof(float32 - 2.0),
arrow_typeof(float64 - 2.0),
arrow_typeof(decimal - 2.0)
from numeric_types;
----
Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64

############### Multiplication ###############

# Multiply with the same operand type, expect the same output type
# except for decimal which is promoted to the highest precision
query TTTTTTTTTTT
Expand All @@ -91,6 +168,45 @@ from numeric_types;
----
Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 Float32 Float64 Decimal128(11, 4)

# Multiply with literal integer
query TTTTTTTTTTT
select
arrow_typeof(int8 * 2),
arrow_typeof(int16 * 2),
arrow_typeof(int32 * 2),
arrow_typeof(int64 * 2),
arrow_typeof(uint8 * 2),
arrow_typeof(uint16 * 2),
arrow_typeof(uint32 * 2),
arrow_typeof(uint64 * 2),
arrow_typeof(float32 * 2),
arrow_typeof(float64 * 2),
arrow_typeof(decimal * 2)
from numeric_types;
----
Int64 Int64 Int64 Int64 Int64 Int64 Int64 Int64 Float32 Float64 Decimal128(26, 2)

# Multiply with literal decimal
query TTTTTTTTTTT
select
arrow_typeof(int8 * 2.0),
arrow_typeof(int16 * 2.0),
arrow_typeof(int32 * 2.0),
arrow_typeof(int64 * 2.0),
arrow_typeof(uint8 * 2.0),
arrow_typeof(uint16 * 2.0),
arrow_typeof(uint32 * 2.0),
arrow_typeof(uint64 * 2.0),
arrow_typeof(float32 * 2.0),
arrow_typeof(float64 * 2.0),
arrow_typeof(decimal * 2.0)
from numeric_types;
----
Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64

############### Division ###############


# Divide with the same operand type, expect the same output type
# except for decimal which is promoted to the highest precision
query TTTTTTTTTTT
Expand All @@ -110,5 +226,139 @@ from numeric_types;
----
Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 Float32 Float64 Decimal128(11, 6)

# Divide with literal integer
query TTTTTTTTTTT
select
arrow_typeof(int8 / 2),
arrow_typeof(int16 / 2),
arrow_typeof(int32 / 2),
arrow_typeof(int64 / 2),
arrow_typeof(uint8 / 2),
arrow_typeof(uint16 / 2),
arrow_typeof(uint32 / 2),
arrow_typeof(uint64 / 2),
arrow_typeof(float32 / 2),
arrow_typeof(float64 / 2),
arrow_typeof(decimal / 2)
from numeric_types;
----
Int64 Int64 Int64 Int64 Int64 Int64 Int64 Int64 Float32 Float64 Decimal128(9, 6)

# Divide with literal decimal
query TTTTTTTTTTT
select
arrow_typeof(int8 / 2.0),
arrow_typeof(int16 / 2.0),
arrow_typeof(int32 / 2.0),
arrow_typeof(int64 / 2.0),
arrow_typeof(uint8 / 2.0),
arrow_typeof(uint16 / 2.0),
arrow_typeof(uint32 / 2.0),
arrow_typeof(uint64 / 2.0),
arrow_typeof(float32 / 2.0),
arrow_typeof(float64 / 2.0),
arrow_typeof(decimal / 2.0)
from numeric_types;
----
Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64 Float64

###############
# Test for comparison with constants uses efficient types
# Expect the physical plans to compare with constants of the same type
# should have no casts of the column to a different type

statement ok
set datafusion.explain.physical_plan_only = true;

############### Less Than ###############

## < positive integer (expect no casts)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 < 5 AND uint64 < 5 AND float64 < 5 AND decimal < 5;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: int64@3 < 5 AND uint64@7 < 5 AND float64@9 < 5 AND decimal@10 < Some(500),5,2
Copy link
Contributor Author

@alamb alamb Jan 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test shows that with a positive integer comparison does not result in any casting of the column

03)----MemoryExec: partitions=1, partition_sizes=[1]

## < negative integer (expect no casts)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 < -5 AND uint64 < -5 AND float64 < -5 AND decimal < -5;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: int64@3 < -5 AND CAST(uint64@7 AS Decimal128(20, 0)) < Some(-5),20,0 AND float64@9 < -5 AND decimal@10 < Some(-500),5,2
Copy link
Contributor Author

@alamb alamb Jan 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test show that with a negative integer, the comparison to int64 does not cast.

However, a comparison with -5 does cast to Decimal128: CAST(uint64@7 AS Decimal128(20, 0)) < Some(-5),20,0

When I reverted the changes in #14223 locally, this is the difference (cast to Int64 rather than Decimal):

[Diff] (-expected|+actual)
    physical_plan
    01)CoalesceBatchesExec: target_batch_size=8192
-   02)--FilterExec: int64@3 < -5 AND CAST(uint64@7 AS Decimal128(20, 0)) < Some(-5),20,0 AND float64@9 < -5 AND decimal@10 < Some(-500),5,2
+   02)--FilterExec: int64@3 < -5 AND CAST(uint64@7 AS Int64) < -5 AND float64@9 < -5 AND decimal@10 < Some(-500),5,2
    03)----MemoryExec: partitions=1, partition_sizes=[1]
at test_files/operator.slt:286

However, note that in this case the predicate will never be true anyways (a uint64 can never be less than -5 as it is by definition always greater than zero -- something @gatesn pointed out on another ticket)

03)----MemoryExec: partitions=1, partition_sizes=[1]

## < decimal (expect casts for integers to float)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 < 5.1 AND uint64 < 5.1 AND float64 < 5.1 AND decimal < 5.1;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: CAST(int64@3 AS Float64) < 5.1 AND CAST(uint64@7 AS Float64) < 5.1 AND float64@9 < 5.1 AND decimal@10 < Some(510),5,2
03)----MemoryExec: partitions=1, partition_sizes=[1]

## < negative decimal (expect casts for integers to float)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 < -5.1 AND uint64 < -5.1 AND float64 < -5.1 AND decimal < -5.1;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: CAST(int64@3 AS Float64) < -5.1 AND CAST(uint64@7 AS Float64) < -5.1 AND float64@9 < -5.1 AND decimal@10 < Some(-510),5,2
03)----MemoryExec: partitions=1, partition_sizes=[1]


############### Equality ###############

## = positive integer (expect no casts)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 = 5 AND uint64 = 5 AND float64 = 5 AND decimal = 5;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: int64@3 = 5 AND uint64@7 = 5 AND float64@9 = 5 AND decimal@10 = Some(500),5,2
03)----MemoryExec: partitions=1, partition_sizes=[1]

## = negative integer (expect no casts)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 = -5 AND uint64 = -5 AND float64 = -5 AND decimal = -5;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: int64@3 = -5 AND CAST(uint64@7 AS Decimal128(20, 0)) = Some(-5),20,0 AND float64@9 = -5 AND decimal@10 = Some(-500),5,2
03)----MemoryExec: partitions=1, partition_sizes=[1]

## = decimal (expect casts for integers to float)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 = 5.1 AND uint64 = 5.1 AND float64 = 5.1 AND decimal = 5.1;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: CAST(int64@3 AS Float64) = 5.1 AND CAST(uint64@7 AS Float64) = 5.1 AND float64@9 = 5.1 AND decimal@10 = Some(510),5,2
03)----MemoryExec: partitions=1, partition_sizes=[1]

## = negative decimal (expect casts for integers to float)
query TT
EXPLAIN SELECT * FROM numeric_types
WHERE int64 = -5.1 AND uint64 = -5.1 AND float64 = -5.1 AND decimal = -5.1;
----
physical_plan
01)CoalesceBatchesExec: target_batch_size=8192
02)--FilterExec: CAST(int64@3 AS Float64) = -5.1 AND CAST(uint64@7 AS Float64) = -5.1 AND float64@9 = -5.1 AND decimal@10 = Some(-510),5,2
03)----MemoryExec: partitions=1, partition_sizes=[1]


statement ok
set datafusion.explain.physical_plan_only = false;


statement ok
drop table numeric_types
Loading