Skip to content

Migrate Write sub-classes in spark-extensions to JUnit5 and AssertJ style #9670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nastra merged 11 commits into from
Feb 20, 2024
Merged

Migrate Write sub-classes in spark-extensions to JUnit5 and AssertJ style #9670

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e069cee
Migrate Write sub-classes in spark-extensions to JUnit5 and AssertJ s…
tomtongue Feb 7, 2024
4f8c57d
Update TestDelete related sub-classes with JUnit 5 and AssertJ
tomtongue Feb 11, 2024
d7e2e82
Update TestUpdate related sub-classes with JUnit 5 and AssertJ
tomtongue Feb 12, 2024
d3460dc
Update TestMerge related sub-classes and remove isTrue or isFalse ass…
tomtongue Feb 13, 2024
a468369
Change the sleep duration to avoid data confliction
tomtongue Feb 14, 2024
f05d850
Fix list validation
tomtongue Feb 14, 2024
184473f
Fix confliction error
tomtongue Feb 15, 2024
abb8f62
Update Thread.sleep to Awaitility
tomtongue Feb 18, 2024
d81094c
Make the temp file name randomized and update other sleeps with Await…
tomtongue Feb 19, 2024
74c6d92
Nit: remove one unnecessary line
tomtongue Feb 19, 2024
099fd31
Add timeout to avoid long time wait
tomtongue Feb 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 38 additions & 44 deletions ...ns/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
import org.apache.iceberg.Parameter;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
import org.apache.iceberg.PlanningMode;
import org.apache.iceberg.RowLevelOperationMode;
import org.apache.iceberg.Snapshot;
Expand All @@ -69,41 +72,30 @@
import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
import org.apache.spark.sql.execution.SparkPlan;
import org.junit.Assert;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.junit.jupiter.api.extension.ExtendWith;

@RunWith(Parameterized.class)
public abstract class SparkRowLevelOperationsTestBase extends SparkExtensionsTestBase {
@ExtendWith(ParameterizedTestExtension.class)
public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase {

private static final Random RANDOM = ThreadLocalRandom.current();

protected final String fileFormat;
protected final boolean vectorized;
protected final String distributionMode;
protected final boolean fanoutEnabled;
protected final String branch;
protected final PlanningMode planningMode;

public SparkRowLevelOperationsTestBase(
String catalogName,
String implementation,
Map<String, String> config,
String fileFormat,
boolean vectorized,
String distributionMode,
boolean fanoutEnabled,
String branch,
PlanningMode planningMode) {
super(catalogName, implementation, config);
this.fileFormat = fileFormat;
this.vectorized = vectorized;
this.distributionMode = distributionMode;
this.fanoutEnabled = fanoutEnabled;
this.branch = branch;
this.planningMode = planningMode;
}
@Parameter(index = 3)
protected String fileFormat;

@nastra nastra Feb 13, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
protected String fileFormat;
protected FileFormat fileFormat;

@tomtongue tomtongue Feb 13, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree. Let me do this.


@Parameter(index = 4)
protected boolean vectorized;

@Parameter(index = 5)
protected String distributionMode;

@Parameter(index = 6)
protected boolean fanoutEnabled;

@Parameter(index = 7)
protected String branch;

@Parameter(index = 8)
protected PlanningMode planningMode;

@Parameters(
name =
Expand Down Expand Up @@ -199,7 +191,7 @@ protected void initTable() {
tableName, ORC_VECTORIZATION_ENABLED, vectorized);
break;
case "avro":
Assert.assertFalse(vectorized);
assertThat(vectorized).isFalse();
break;
}

Expand Down Expand Up @@ -303,7 +295,7 @@ protected void validateSnapshot(
String deletedDataFiles,
String addedDeleteFiles,
String addedDataFiles) {
Assert.assertEquals("Operation must match", operation, snapshot.operation());
assertThat(snapshot.operation()).as("Operation must match").isEqualTo(operation);
validateProperty(snapshot, CHANGED_PARTITION_COUNT_PROP, changedPartitionCount);
validateProperty(snapshot, DELETED_FILES_PROP, deletedDataFiles);
validateProperty(snapshot, ADDED_DELETE_FILES_PROP, addedDeleteFiles);
Expand All @@ -312,20 +304,22 @@ protected void validateSnapshot(

protected void validateProperty(Snapshot snapshot, String property, Set<String> expectedValues) {
String actual = snapshot.summary().get(property);
Assert.assertTrue(
"Snapshot property "
+ property
+ " has unexpected value, actual = "
+ actual
+ ", expected one of : "
+ String.join(",", expectedValues),
expectedValues.contains(actual));
assertThat(expectedValues)
.as(
"Snapshot property "
+ property
+ " has unexpected value, actual = "
+ actual
+ ", expected one of : "
+ String.join(",", expectedValues))
.contains(actual);
}

protected void validateProperty(Snapshot snapshot, String property, String expectedValue) {
String actual = snapshot.summary().get(property);
Assert.assertEquals(
"Snapshot property " + property + " has unexpected value.", expectedValue, actual);
assertThat(actual)
.as("Snapshot property " + property + " has unexpected value.")
.isEqualTo(expectedValue);
}

protected void sleep(long millis) {
Expand All @@ -338,7 +332,7 @@ protected void sleep(long millis) {

protected DataFile writeDataFile(Table table, List<GenericRecord> records) {
try {
OutputFile file = Files.localOutput(temp.newFile());
OutputFile file = Files.localOutput(temp.toFile());

DataWriter<GenericRecord> dataWriter =
Parquet.writeData(file)
Expand Down
43 changes: 20 additions & 23 deletions ...-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestConflictValidation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
package org.apache.iceberg.spark.extensions;

import java.util.List;
import java.util.Map;
import org.apache.iceberg.IsolationLevel;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Table;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
Expand All @@ -30,18 +30,15 @@
import org.apache.spark.sql.Row;
import org.apache.spark.sql.functions;
import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;

public class TestConflictValidation extends SparkExtensionsTestBase {
@ExtendWith(ParameterizedTestExtension.class)
public class TestConflictValidation extends ExtensionsTestBase {

public TestConflictValidation(
String catalogName, String implementation, Map<String, String> config) {
super(catalogName, implementation, config);
}

@Before
@BeforeEach
public void createTables() {
sql(
"CREATE TABLE %s (id int, data string) USING iceberg "
Expand All @@ -53,12 +50,12 @@ public void createTables() {
sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName);
}

@After
@AfterEach
public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}

@Test
@TestTemplate
public void testOverwriteFilterSerializableIsolation() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);
long snapshotId = table.currentSnapshot().snapshotId();
Expand Down Expand Up @@ -90,7 +87,7 @@ public void testOverwriteFilterSerializableIsolation() throws Exception {
.overwrite(functions.col("id").equalTo(1));
}

@Test
@TestTemplate
public void testOverwriteFilterSerializableIsolation2() throws Exception {
List<SimpleRecord> records =
Lists.newArrayList(new SimpleRecord(1, "a"), new SimpleRecord(1, "b"));
Expand Down Expand Up @@ -127,7 +124,7 @@ public void testOverwriteFilterSerializableIsolation2() throws Exception {
.overwrite(functions.col("id").equalTo(1));
}

@Test
@TestTemplate
public void testOverwriteFilterSerializableIsolation3() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);
long snapshotId = table.currentSnapshot().snapshotId();
Expand Down Expand Up @@ -161,7 +158,7 @@ public void testOverwriteFilterSerializableIsolation3() throws Exception {
.overwrite(functions.col("id").equalTo(1));
}

@Test
@TestTemplate
public void testOverwriteFilterNoSnapshotIdValidation() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);

Expand Down Expand Up @@ -192,7 +189,7 @@ public void testOverwriteFilterNoSnapshotIdValidation() throws Exception {
.overwrite(functions.col("id").equalTo(1));
}

@Test
@TestTemplate
public void testOverwriteFilterSnapshotIsolation() throws Exception {
List<SimpleRecord> records =
Lists.newArrayList(new SimpleRecord(1, "a"), new SimpleRecord(1, "b"));
Expand Down Expand Up @@ -229,7 +226,7 @@ public void testOverwriteFilterSnapshotIsolation() throws Exception {
.overwrite(functions.col("id").equalTo(1));
}

@Test
@TestTemplate
public void testOverwriteFilterSnapshotIsolation2() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);
long snapshotId = table.currentSnapshot().snapshotId();
Expand All @@ -246,7 +243,7 @@ public void testOverwriteFilterSnapshotIsolation2() throws Exception {
.overwrite(functions.col("id").equalTo(1));
}

@Test
@TestTemplate
public void testOverwritePartitionSerializableIsolation() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);
final long snapshotId = table.currentSnapshot().snapshotId();
Expand Down Expand Up @@ -278,7 +275,7 @@ public void testOverwritePartitionSerializableIsolation() throws Exception {
.overwritePartitions();
}

@Test
@TestTemplate
public void testOverwritePartitionSnapshotIsolation() throws Exception {
List<SimpleRecord> records =
Lists.newArrayList(new SimpleRecord(1, "a"), new SimpleRecord(1, "b"));
Expand Down Expand Up @@ -313,7 +310,7 @@ public void testOverwritePartitionSnapshotIsolation() throws Exception {
.overwritePartitions();
}

@Test
@TestTemplate
public void testOverwritePartitionSnapshotIsolation2() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);
final long snapshotId = table.currentSnapshot().snapshotId();
Expand Down Expand Up @@ -347,7 +344,7 @@ public void testOverwritePartitionSnapshotIsolation2() throws Exception {
.overwritePartitions();
}

@Test
@TestTemplate
public void testOverwritePartitionSnapshotIsolation3() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);
final long snapshotId = table.currentSnapshot().snapshotId();
Expand All @@ -364,7 +361,7 @@ public void testOverwritePartitionSnapshotIsolation3() throws Exception {
.overwritePartitions();
}

@Test
@TestTemplate
public void testOverwritePartitionNoSnapshotIdValidation() throws Exception {
Table table = validationCatalog.loadTable(tableIdent);

Expand Down
43 changes: 11 additions & 32 deletions ...k-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
package org.apache.iceberg.spark.extensions;

import static org.apache.iceberg.TableProperties.DELETE_ISOLATION_LEVEL;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assumptions.assumeThat;

import java.util.Collections;
import java.util.Map;
Expand All @@ -32,15 +34,14 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.iceberg.AppendFiles;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.PlanningMode;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.RowLevelOperationMode;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors;
import org.apache.iceberg.spark.Spark3Util;
import org.apache.iceberg.spark.SparkSQLProperties;
Expand All @@ -49,45 +50,23 @@
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
import org.apache.spark.sql.internal.SQLConf;
import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;

@ExtendWith(ParameterizedTestExtension.class)
public class TestCopyOnWriteDelete extends TestDelete {

public TestCopyOnWriteDelete(
String catalogName,
String implementation,
Map<String, String> config,
String fileFormat,
Boolean vectorized,
String distributionMode,
boolean fanoutEnabled,
String branch,
PlanningMode planningMode) {
super(
catalogName,
implementation,
config,
fileFormat,
vectorized,
distributionMode,
fanoutEnabled,
branch,
planningMode);
}

@Override
protected Map<String, String> extraTableProperties() {
return ImmutableMap.of(
TableProperties.DELETE_MODE, RowLevelOperationMode.COPY_ON_WRITE.modeName());
}

@Test
@TestTemplate
public synchronized void testDeleteWithConcurrentTableRefresh() throws Exception {
// this test can only be run with Hive tables as it requires a reliable lock
// also, the table cache must be enabled so that the same table instance can be reused
Assume.assumeTrue(catalogName.equalsIgnoreCase("testhive"));
assumeThat(catalogName.equalsIgnoreCase("testhive")).isTrue();

createAndInitUnpartitionedTable();
createOrReplaceView("deleted_id", Collections.singletonList(1), Encoders.INT());
Expand Down Expand Up @@ -167,10 +146,10 @@ public synchronized void testDeleteWithConcurrentTableRefresh() throws Exception
}

executorService.shutdown();
Assert.assertTrue("Timeout", executorService.awaitTermination(2, TimeUnit.MINUTES));
assertThat(executorService.awaitTermination(2, TimeUnit.MINUTES)).as("Timeout").isTrue();
}

@Test
@TestTemplate
public void testRuntimeFilteringWithPreservedDataGrouping() throws NoSuchTableException {
createAndInitPartitionedTable();

Expand All @@ -188,7 +167,7 @@ public void testRuntimeFilteringWithPreservedDataGrouping() throws NoSuchTableEx
withSQLConf(sqlConf, () -> sql("DELETE FROM %s WHERE id = 2", commitTarget()));

Table table = validationCatalog.loadTable(tableIdent);
Assert.assertEquals("Should have 3 snapshots", 3, Iterables.size(table.snapshots()));
assertThat(table.snapshots()).as("Should have 3 snapshots").hasSize(3);

Snapshot currentSnapshot = SnapshotUtil.latestSnapshot(table, branch);
validateCopyOnWrite(currentSnapshot, "1", "1", "1");
Expand Down
Loading