Skip to content

Fix: Align sort_merge_join filter output with join schema to fix right-anti panic #18800

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rluvaton merged 5 commits into from
Dec 4, 2025
Merged

Fix: Align sort_merge_join filter output with join schema to fix right-anti panic #18800

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0d7a9e8
Fix: Align sort_merge_join filter output with join schema to fix righ…
kumarUjjawal Nov 18, 2025
537e70f
Merge branch 'main' into fix/sort-merge-join
kumarUjjawal Nov 19, 2025
da847c4
added unit tests and updated fuzz
kumarUjjawal Nov 19, 2025
08d1e8e
remove .take
rluvaton Dec 3, 2025
951a9b1
Merge branch 'main' into fix/sort-merge-join
rluvaton Dec 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 40 additions & 32 deletions datafusion/core/tests/fuzz_cases/join_fuzz.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,14 +245,16 @@ async fn test_right_semi_join_1k() {

#[tokio::test]
async fn test_right_semi_join_1k_filtered() {
JoinFuzzTestCase::new(
make_staggered_batches_i32(1000, false),
make_staggered_batches_i32(1000, false),
JoinType::RightSemi,
Some(Box::new(col_lt_col_filter)),
)
.run_test(&[HjSmj, NljHj], false)
.await
for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
JoinFuzzTestCase::new(
make_staggered_batches_i32(1000, left_extra),
make_staggered_batches_i32(1000, right_extra),
JoinType::RightSemi,
Some(Box::new(col_lt_col_filter)),
)
.run_test(&[HjSmj, NljHj], false)
.await
}
}

#[tokio::test]
Expand Down Expand Up @@ -299,14 +301,16 @@ async fn test_right_anti_join_1k() {

#[tokio::test]
async fn test_right_anti_join_1k_filtered() {
JoinFuzzTestCase::new(
make_staggered_batches_i32(1000, false),
make_staggered_batches_i32(1000, false),
JoinType::RightAnti,
Some(Box::new(col_lt_col_filter)),
)
.run_test(&[HjSmj, NljHj], false)
.await
for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
JoinFuzzTestCase::new(
make_staggered_batches_i32(1000, left_extra),
make_staggered_batches_i32(1000, right_extra),
JoinType::RightAnti,
Some(Box::new(col_lt_col_filter)),
)
.run_test(&[HjSmj, NljHj], false)
.await
}
}

#[tokio::test]
Expand Down Expand Up @@ -564,26 +568,30 @@ async fn test_left_anti_join_1k_binary_filtered() {

#[tokio::test]
async fn test_right_anti_join_1k_binary() {
JoinFuzzTestCase::new(
make_staggered_batches_binary(1000, false),
make_staggered_batches_binary(1000, false),
JoinType::RightAnti,
None,
)
.run_test(&[HjSmj, NljHj], false)
.await
for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
JoinFuzzTestCase::new(
make_staggered_batches_binary(1000, left_extra),
make_staggered_batches_binary(1000, right_extra),
JoinType::RightAnti,
None,
)
.run_test(&[HjSmj, NljHj], false)
.await
}
}

#[tokio::test]
async fn test_right_anti_join_1k_binary_filtered() {
JoinFuzzTestCase::new(
make_staggered_batches_binary(1000, false),
make_staggered_batches_binary(1000, false),
JoinType::RightAnti,
Some(Box::new(col_lt_col_filter)),
)
.run_test(&[HjSmj, NljHj], false)
.await
for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
JoinFuzzTestCase::new(
make_staggered_batches_binary(1000, left_extra),
make_staggered_batches_binary(1000, right_extra),
JoinType::RightAnti,
Some(Box::new(col_lt_col_filter)),
)
.run_test(&[HjSmj, NljHj], false)
.await
}
}

#[tokio::test]
Expand Down
13 changes: 8 additions & 5 deletions datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1566,6 +1566,7 @@ impl SortMergeJoinStream {
.columns()
.iter()
.skip(right_columns_length)
.take(left_columns_length)

@rluvaton rluvaton Nov 19, 2025 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we should add it as we can lose columns or hide bugs if we the rest of the columns are not left

@kumarUjjawal kumarUjjawal Nov 19, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, should I Drop the .take(left_columns_length) and add a sanity check like assert_eq_or_internal_err!(..., left_columns_length, "...")) before extending right_columns.

@rluvaton rluvaton Dec 3, 2025

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes

.cloned()
.collect::<Vec<_>>();

Expand Down Expand Up @@ -1595,14 +1596,16 @@ impl SortMergeJoinStream {
&self.schema,
&[filtered_record_batch, null_joined_streamed_batch],
)?;
} else if matches!(self.join_type, JoinType::LeftSemi | JoinType::LeftAnti) {
} else if matches!(
self.join_type,
JoinType::LeftSemi
| JoinType::LeftAnti
| JoinType::RightAnti
| JoinType::RightSemi
) {
let output_column_indices = (0..left_columns_length).collect::<Vec<_>>();
filtered_record_batch =
filtered_record_batch.project(&output_column_indices)?;
} else if matches!(self.join_type, JoinType::RightAnti | JoinType::RightSemi) {
let output_column_indices = (0..right_columns_length).collect::<Vec<_>>();
filtered_record_batch =
filtered_record_batch.project(&output_column_indices)?;
} else if matches!(self.join_type, JoinType::Full)
&& corrected_mask.false_count() > 0
{
Expand Down
55 changes: 55 additions & 0 deletions datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1109,6 +1109,61 @@ async fn join_right_anti_two_with_filter() -> Result<()> {
Ok(())
}

#[tokio::test]
async fn join_right_anti_filtered_with_mismatched_columns() -> Result<()> {
let left = build_table_two_cols(("a1", &vec![31, 31]), ("b1", &vec![32, 33]));
let right = build_table(
("a2", &vec![31, 31]),
("b2", &vec![32, 35]),
("c2", &vec![108, 109]),
);
let on = vec![
(
Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,
Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,
),
(
Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,
),
];

let filter = JoinFilter::new(
Arc::new(BinaryExpr::new(
Arc::new(Column::new("b1", 0)),
Operator::LtEq,
Arc::new(Column::new("c2", 1)),
)),
vec![
ColumnIndex {
index: 1,
side: JoinSide::Left,
},
ColumnIndex {
index: 2,
side: JoinSide::Right,
},
],
Arc::new(Schema::new(vec![
Field::new("b1", DataType::Int32, false),
Field::new("c2", DataType::Int32, false),
])),
);

let (_, batches) =
join_collect_with_filter(left, right, on, filter, RightAnti).await?;

let expected = [
"+----+----+-----+",
"| a2 | b2 | c2 |",
"+----+----+-----+",
"| 31 | 35 | 109 |",
"+----+----+-----+",
];
assert_batches_eq!(expected, &batches);
Ok(())
}

#[tokio::test]
async fn join_right_anti_with_nulls() -> Result<()> {
let left = build_table_i32_nullable(
Expand Down