diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index d4cb67a2bbf3..b29ba6761ebc 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -239,6 +239,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark.hive35.get()}" integrationImplementation libs.junit.vintage.engine + integrationImplementation libs.junit.jupiter integrationImplementation libs.slf4j.simple integrationImplementation libs.assertj.core integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') @@ -288,6 +289,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio } task integrationTest(type: Test) { + useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" group = "verification" jvmArgs += project.property('extraJvmArgs') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java index 91126776528a..3b2427eea7ca 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java @@ -24,7 +24,6 @@ import org.apache.iceberg.Table; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.assertj.core.api.Assertions; -import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -74,8 +73,9 @@ public void testSetIdentifierFields() { public void testSetInvalidIdentifierFields() { sql("CREATE TABLE %s (id bigint NOT NULL, id2 bigint) USING iceberg", tableName); Table table = validationCatalog.loadTable(tableIdent); - Assert.assertTrue( - "Table should start without identifier", table.schema().identifierFieldIds().isEmpty()); + assertThat(table.schema().identifierFieldIds()) + .as("Table should start without identifier") + .isEmpty(); Assertions.assertThatThrownBy( () -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS unknown", tableName)) .isInstanceOf(IllegalArgumentException.class) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java index e9013848cf11..b783a006ef73 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java @@ -18,9 +18,12 @@ */ package org.apache.iceberg.spark.extensions; +import static org.assertj.core.api.Assertions.assertThat; + import java.util.List; -import java.util.Map; import java.util.UUID; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; import org.apache.iceberg.ScanTask; import org.apache.iceberg.Table; import org.apache.iceberg.io.CloseableIterable; @@ -34,19 +37,14 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; -import org.assertj.core.api.Assertions; -import org.junit.After; -import org.junit.Test; -import org.junit.runners.Parameterized; - -public class TestMetaColumnProjectionWithStageScan extends SparkExtensionsTestBase { +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; - public TestMetaColumnProjectionWithStageScan( - String catalogName, String implementation, Map config) { - super(catalogName, implementation, config); - } +@ExtendWith(ParameterizedTestExtension.class) +public class TestMetaColumnProjectionWithStageScan extends ExtensionsTestBase { - @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") public static Object[][] parameters() { return new Object[][] { { @@ -57,7 +55,7 @@ public static Object[][] parameters() { }; } - @After + @AfterEach public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } @@ -68,7 +66,7 @@ private void stageTask( taskSetManager.stageTasks(tab, fileSetID, Lists.newArrayList(tasks)); } - @Test + @TestTemplate public void testReadStageTableMeta() throws Exception { sql( "CREATE TABLE %s (id bigint, data string) USING iceberg TBLPROPERTIES" @@ -103,7 +101,7 @@ public void testReadStageTableMeta() throws Exception { .option(SparkReadOptions.SCAN_TASK_SET_ID, fileSetID) .load(tableLocation); - Assertions.assertThat(scanDF2.columns().length).isEqualTo(2); + assertThat(scanDF2.columns()).hasSize(2); } try (CloseableIterable tasks = table.newBatchScan().planFiles()) { diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java index 50376589b671..a22cf61ec8c9 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.avro.generic.GenericData.Record; @@ -33,6 +32,7 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.HistoryEntry; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -53,22 +53,19 @@ import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.catalyst.util.DateTimeUtils; import org.apache.spark.sql.types.StructType; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; -public class TestMetadataTables extends SparkExtensionsTestBase { +@ExtendWith(ParameterizedTestExtension.class) +public class TestMetadataTables extends ExtensionsTestBase { - public TestMetadataTables(String catalogName, String implementation, Map config) { - super(catalogName, implementation, config); - } - - @After + @AfterEach public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } - @Test + @TestTemplate public void testUnpartitionedTable() throws Exception { sql( "CREATE TABLE %s (id bigint, data string) USING iceberg TBLPROPERTIES" @@ -92,8 +89,8 @@ public void testUnpartitionedTable() throws Exception { Table table = Spark3Util.loadIcebergTable(spark, tableName); List expectedDataManifests = TestHelpers.dataManifests(table); List expectedDeleteManifests = TestHelpers.deleteManifests(table); - Assert.assertEquals("Should have 1 data manifest", 1, expectedDataManifests.size()); - Assert.assertEquals("Should have 1 delete manifest", 1, expectedDeleteManifests.size()); + assertThat(expectedDataManifests).as("Should have 1 data manifest").hasSize(1); + assertThat(expectedDeleteManifests).as("Should have 1 delete manifest").hasSize(1); Schema entriesTableSchema = Spark3Util.loadIcebergTable(spark, tableName + ".entries").schema(); Schema filesTableSchema = Spark3Util.loadIcebergTable(spark, tableName + ".files").schema(); @@ -101,13 +98,12 @@ public void testUnpartitionedTable() throws Exception { // check delete files table Dataset actualDeleteFilesDs = spark.sql("SELECT * FROM " + tableName + ".delete_files"); List actualDeleteFiles = TestHelpers.selectNonDerived(actualDeleteFilesDs).collectAsList(); - Assert.assertEquals( - "Metadata table should return one delete file", 1, actualDeleteFiles.size()); + assertThat(actualDeleteFiles).as("Metadata table should return one delete file").hasSize(1); List expectedDeleteFiles = expectedEntries( table, FileContent.POSITION_DELETES, entriesTableSchema, expectedDeleteManifests, null); - Assert.assertEquals("Should be one delete file manifest entry", 1, expectedDeleteFiles.size()); + assertThat(expectedDeleteFiles).as("Should be one delete file manifest entry").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDeleteFilesDs), expectedDeleteFiles.get(0), @@ -116,11 +112,11 @@ public void testUnpartitionedTable() throws Exception { // check data files table Dataset actualDataFilesDs = spark.sql("SELECT * FROM " + tableName + ".data_files"); List actualDataFiles = TestHelpers.selectNonDerived(actualDataFilesDs).collectAsList(); - Assert.assertEquals("Metadata table should return one data file", 1, actualDataFiles.size()); + assertThat(actualDataFiles).as("Metadata table should return one data file").hasSize(1); List expectedDataFiles = expectedEntries(table, FileContent.DATA, entriesTableSchema, expectedDataManifests, null); - Assert.assertEquals("Should be one data file manifest entry", 1, expectedDataFiles.size()); + assertThat(expectedDataFiles).as("Should be one data file manifest entry").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDataFilesDs), expectedDataFiles.get(0), @@ -131,19 +127,19 @@ public void testUnpartitionedTable() throws Exception { spark.sql("SELECT * FROM " + tableName + ".files ORDER BY content"); List actualFiles = TestHelpers.selectNonDerived(actualFilesDs).collectAsList(); - Assert.assertEquals("Metadata table should return two files", 2, actualFiles.size()); + assertThat(actualFiles).as("Metadata table should return two files").hasSize(2); List expectedFiles = Stream.concat(expectedDataFiles.stream(), expectedDeleteFiles.stream()) .collect(Collectors.toList()); - Assert.assertEquals("Should have two files manifest entries", 2, expectedFiles.size()); + assertThat(expectedFiles).as("Should have two files manifest entries").hasSize(2); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(0), actualFiles.get(0)); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(1), actualFiles.get(1)); } - @Test + @TestTemplate public void testPartitionedTable() throws Exception { sql( "CREATE TABLE %s (id bigint, data string) " @@ -177,8 +173,8 @@ public void testPartitionedTable() throws Exception { List expectedDataManifests = TestHelpers.dataManifests(table); List expectedDeleteManifests = TestHelpers.deleteManifests(table); - Assert.assertEquals("Should have 2 data manifests", 2, expectedDataManifests.size()); - Assert.assertEquals("Should have 2 delete manifests", 2, expectedDeleteManifests.size()); + assertThat(expectedDataManifests).as("Should have 2 data manifest").hasSize(2); + assertThat(expectedDeleteManifests).as("Should have 2 delete manifest").hasSize(2); Schema filesTableSchema = Spark3Util.loadIcebergTable(spark, tableName + ".delete_files").schema(); @@ -187,15 +183,13 @@ public void testPartitionedTable() throws Exception { List expectedDeleteFiles = expectedEntries( table, FileContent.POSITION_DELETES, entriesTableSchema, expectedDeleteManifests, "a"); - Assert.assertEquals( - "Should have one delete file manifest entry", 1, expectedDeleteFiles.size()); + assertThat(expectedDeleteFiles).as("Should have one delete file manifest entry").hasSize(1); Dataset actualDeleteFilesDs = spark.sql("SELECT * FROM " + tableName + ".delete_files " + "WHERE partition.data='a'"); List actualDeleteFiles = actualDeleteFilesDs.collectAsList(); - Assert.assertEquals( - "Metadata table should return one delete file", 1, actualDeleteFiles.size()); + assertThat(actualDeleteFiles).as("Metadata table should return one delete file").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDeleteFilesDs), expectedDeleteFiles.get(0), @@ -204,13 +198,13 @@ public void testPartitionedTable() throws Exception { // Check data files table List expectedDataFiles = expectedEntries(table, FileContent.DATA, entriesTableSchema, expectedDataManifests, "a"); - Assert.assertEquals("Should have one data file manifest entry", 1, expectedDataFiles.size()); + assertThat(expectedDataFiles).as("Should have one data file manifest entry").hasSize(1); Dataset actualDataFilesDs = spark.sql("SELECT * FROM " + tableName + ".data_files " + "WHERE partition.data='a'"); List actualDataFiles = TestHelpers.selectNonDerived(actualDataFilesDs).collectAsList(); - Assert.assertEquals("Metadata table should return one data file", 1, actualDataFiles.size()); + assertThat(actualDataFiles).as("Metadata table should return one data file").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDataFilesDs), expectedDataFiles.get(0), @@ -218,32 +212,29 @@ public void testPartitionedTable() throws Exception { List actualPartitionsWithProjection = spark.sql("SELECT file_count FROM " + tableName + ".partitions ").collectAsList(); - Assert.assertEquals( - "Metadata table should return two partitions record", - 2, - actualPartitionsWithProjection.size()); - for (int i = 0; i < 2; ++i) { - Assert.assertEquals(1, actualPartitionsWithProjection.get(i).get(0)); - } + assertThat(actualPartitionsWithProjection) + .as("Metadata table should return two partitions record") + .hasSize(2) + .containsExactly(RowFactory.create(1), RowFactory.create(1)); // Check files table List expectedFiles = Stream.concat(expectedDataFiles.stream(), expectedDeleteFiles.stream()) .collect(Collectors.toList()); - Assert.assertEquals("Should have two file manifest entries", 2, expectedFiles.size()); + assertThat(expectedFiles).as("Should have two file manifest entries").hasSize(2); Dataset actualFilesDs = spark.sql( "SELECT * FROM " + tableName + ".files " + "WHERE partition.data='a' ORDER BY content"); List actualFiles = TestHelpers.selectNonDerived(actualFilesDs).collectAsList(); - Assert.assertEquals("Metadata table should return two files", 2, actualFiles.size()); + assertThat(actualFiles).as("Metadata table should return two files").hasSize(2); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(0), actualFiles.get(0)); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(1), actualFiles.get(1)); } - @Test + @TestTemplate public void testAllFilesUnpartitioned() throws Exception { sql( "CREATE TABLE %s (id bigint, data string) USING iceberg TBLPROPERTIES" @@ -267,13 +258,13 @@ public void testAllFilesUnpartitioned() throws Exception { Table table = Spark3Util.loadIcebergTable(spark, tableName); List expectedDataManifests = TestHelpers.dataManifests(table); - Assert.assertEquals("Should have 1 data manifest", 1, expectedDataManifests.size()); + assertThat(expectedDataManifests).as("Should have 1 data manifest").hasSize(1); List expectedDeleteManifests = TestHelpers.deleteManifests(table); - Assert.assertEquals("Should have 1 delete manifest", 1, expectedDeleteManifests.size()); + assertThat(expectedDeleteManifests).as("Should have 1 delete manifest").hasSize(1); // Clear table to test whether 'all_files' can read past files List results = sql("DELETE FROM %s", tableName); - Assert.assertEquals("Table should be cleared", 0, results.size()); + assertThat(results).as("Table should be cleared").isEmpty(); Schema entriesTableSchema = Spark3Util.loadIcebergTable(spark, tableName + ".entries").schema(); Schema filesTableSchema = @@ -285,8 +276,8 @@ public void testAllFilesUnpartitioned() throws Exception { List expectedDataFiles = expectedEntries(table, FileContent.DATA, entriesTableSchema, expectedDataManifests, null); - Assert.assertEquals("Should be one data file manifest entry", 1, expectedDataFiles.size()); - Assert.assertEquals("Metadata table should return one data file", 1, actualDataFiles.size()); + assertThat(expectedDataFiles).as("Should be one data file manifest entry").hasSize(1); + assertThat(actualDataFiles).as("Metadata table should return one data file").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDataFilesDs), expectedDataFiles.get(0), @@ -299,9 +290,8 @@ public void testAllFilesUnpartitioned() throws Exception { List expectedDeleteFiles = expectedEntries( table, FileContent.POSITION_DELETES, entriesTableSchema, expectedDeleteManifests, null); - Assert.assertEquals("Should be one delete file manifest entry", 1, expectedDeleteFiles.size()); - Assert.assertEquals( - "Metadata table should return one delete file", 1, actualDeleteFiles.size()); + assertThat(expectedDeleteFiles).as("Should be one delete file manifest entry").hasSize(1); + assertThat(actualDeleteFiles).as("Metadata table should return one delete file").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDeleteFilesDs), expectedDeleteFiles.get(0), @@ -313,12 +303,12 @@ public void testAllFilesUnpartitioned() throws Exception { List actualFiles = actualFilesDs.collectAsList(); List expectedFiles = ListUtils.union(expectedDataFiles, expectedDeleteFiles); expectedFiles.sort(Comparator.comparing(r -> ((Integer) r.get("content")))); - Assert.assertEquals("Metadata table should return two files", 2, actualFiles.size()); + assertThat(actualFiles).as("Metadata table should return two files").hasSize(2); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles, actualFiles); } - @Test + @TestTemplate public void testAllFilesPartitioned() throws Exception { // Create table and insert data sql( @@ -350,13 +340,13 @@ public void testAllFilesPartitioned() throws Exception { Table table = Spark3Util.loadIcebergTable(spark, tableName); List expectedDataManifests = TestHelpers.dataManifests(table); - Assert.assertEquals("Should have 2 data manifests", 2, expectedDataManifests.size()); + assertThat(expectedDataManifests).as("Should have 2 data manifests").hasSize(2); List expectedDeleteManifests = TestHelpers.deleteManifests(table); - Assert.assertEquals("Should have 1 delete manifest", 1, expectedDeleteManifests.size()); + assertThat(expectedDeleteManifests).as("Should have 1 delete manifest").hasSize(1); // Clear table to test whether 'all_files' can read past files List results = sql("DELETE FROM %s", tableName); - Assert.assertEquals("Table should be cleared", 0, results.size()); + assertThat(results).as("Table should be cleared").isEmpty(); Schema entriesTableSchema = Spark3Util.loadIcebergTable(spark, tableName + ".entries").schema(); Schema filesTableSchema = @@ -368,8 +358,8 @@ public void testAllFilesPartitioned() throws Exception { List actualDataFiles = TestHelpers.selectNonDerived(actualDataFilesDs).collectAsList(); List expectedDataFiles = expectedEntries(table, FileContent.DATA, entriesTableSchema, expectedDataManifests, "a"); - Assert.assertEquals("Should be one data file manifest entry", 1, expectedDataFiles.size()); - Assert.assertEquals("Metadata table should return one data file", 1, actualDataFiles.size()); + assertThat(expectedDataFiles).as("Should be one data file manifest entry").hasSize(1); + assertThat(actualDataFiles).as("Metadata table should return one data file").hasSize(1); TestHelpers.assertEqualsSafe( SparkSchemaUtil.convert(TestHelpers.selectNonDerived(actualDataFilesDs).schema()) .asStruct(), @@ -384,8 +374,8 @@ public void testAllFilesPartitioned() throws Exception { List expectedDeleteFiles = expectedEntries( table, FileContent.POSITION_DELETES, entriesTableSchema, expectedDeleteManifests, "a"); - Assert.assertEquals("Should be one data file manifest entry", 1, expectedDeleteFiles.size()); - Assert.assertEquals("Metadata table should return one data file", 1, actualDeleteFiles.size()); + assertThat(expectedDeleteFiles).as("Should be one data file manifest entry").hasSize(1); + assertThat(actualDeleteFiles).as("Metadata table should return one data file").hasSize(1); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDeleteFilesDs), @@ -403,12 +393,12 @@ public void testAllFilesPartitioned() throws Exception { List expectedFiles = ListUtils.union(expectedDataFiles, expectedDeleteFiles); expectedFiles.sort(Comparator.comparing(r -> ((Integer) r.get("content")))); - Assert.assertEquals("Metadata table should return two files", 2, actualFiles.size()); + assertThat(actualFiles).as("Metadata table should return two files").hasSize(2); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualDataFilesDs), expectedFiles, actualFiles); } - @Test + @TestTemplate public void testMetadataLogEntries() throws Exception { // Create table and insert data sql( @@ -465,8 +455,9 @@ public void testMetadataLogEntries() throws Exception { sql( "SELECT * FROM %s.metadata_log_entries WHERE latest_snapshot_id = %s", tableName, currentSnapshotId); - Assert.assertEquals( - "metadataLogEntries table should return 1 row", 1, metadataLogWithFilters.size()); + assertThat(metadataLogWithFilters) + .as("metadataLogEntries table should return 1 row") + .hasSize(1); assertEquals( "Result should match the latest snapshot entry", ImmutableList.of( @@ -487,15 +478,16 @@ public void testMetadataLogEntries() throws Exception { metadataFiles.add(tableMetadata.metadataFileLocation()); List metadataLogWithProjection = sql("SELECT file FROM %s.metadata_log_entries", tableName); - Assert.assertEquals( - "metadataLogEntries table should return 3 rows", 3, metadataLogWithProjection.size()); + assertThat(metadataLogWithProjection) + .as("metadataLogEntries table should return 3 rows") + .hasSize(3); assertEquals( "metadataLog entry should be of same file", metadataFiles.stream().map(this::row).collect(Collectors.toList()), metadataLogWithProjection); } - @Test + @TestTemplate public void testFilesTableTimeTravelWithSchemaEvolution() throws Exception { // Create table and insert data sql( @@ -545,7 +537,7 @@ public void testFilesTableTimeTravelWithSchemaEvolution() throws Exception { List expectedFiles = expectedEntries(table, FileContent.DATA, entriesTableSchema, expectedDataManifests, null); - Assert.assertEquals("actualFiles size should be 2", 2, actualFiles.size()); + assertThat(actualFiles).as("actualFiles size should be 2").hasSize(2); TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(0), actualFiles.get(0)); @@ -553,13 +545,12 @@ public void testFilesTableTimeTravelWithSchemaEvolution() throws Exception { TestHelpers.assertEqualsSafe( TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(1), actualFiles.get(1)); - Assert.assertEquals( - "expectedFiles and actualFiles size should be the same", - actualFiles.size(), - expectedFiles.size()); + assertThat(actualFiles) + .as("expectedFiles and actualFiles size should be the same") + .hasSameSizeAs(expectedFiles); } - @Test + @TestTemplate public void testSnapshotReferencesMetatable() throws Exception { // Create table and insert data sql( @@ -605,43 +596,64 @@ public void testSnapshotReferencesMetatable() throws Exception { .commit(); // Check refs table List references = spark.sql("SELECT * FROM " + tableName + ".refs").collectAsList(); - Assert.assertEquals("Refs table should return 3 rows", 3, references.size()); + assertThat(references).as("Refs table should return 3 rows").hasSize(3); List branches = spark.sql("SELECT * FROM " + tableName + ".refs WHERE type='BRANCH'").collectAsList(); - Assert.assertEquals("Refs table should return 2 branches", 2, branches.size()); + assertThat(branches).as("Refs table should return 2 branches").hasSize(2); List tags = spark.sql("SELECT * FROM " + tableName + ".refs WHERE type='TAG'").collectAsList(); - Assert.assertEquals("Refs table should return 1 tag", 1, tags.size()); + assertThat(tags).as("Refs table should return 1 tag").hasSize(1); // Check branch entries in refs table List mainBranch = spark .sql("SELECT * FROM " + tableName + ".refs WHERE name = 'main' AND type='BRANCH'") .collectAsList(); - Assert.assertEquals("main", mainBranch.get(0).getAs("name")); - Assert.assertEquals("BRANCH", mainBranch.get(0).getAs("type")); - Assert.assertEquals(currentSnapshotId, mainBranch.get(0).getAs("snapshot_id")); + assertThat(mainBranch) + .hasSize(1) + .containsExactly(RowFactory.create("main", "BRANCH", currentSnapshotId, null, null, null)); + assertThat(mainBranch.get(0).schema().fieldNames()) + .containsExactly( + "name", + "type", + "snapshot_id", + "max_reference_age_in_ms", + "min_snapshots_to_keep", + "max_snapshot_age_in_ms"); List testBranch = spark .sql("SELECT * FROM " + tableName + ".refs WHERE name = 'testBranch' AND type='BRANCH'") .collectAsList(); - Assert.assertEquals("testBranch", testBranch.get(0).getAs("name")); - Assert.assertEquals("BRANCH", testBranch.get(0).getAs("type")); - Assert.assertEquals(currentSnapshotId, testBranch.get(0).getAs("snapshot_id")); - Assert.assertEquals(Long.valueOf(10), testBranch.get(0).getAs("max_reference_age_in_ms")); - Assert.assertEquals(Integer.valueOf(20), testBranch.get(0).getAs("min_snapshots_to_keep")); - Assert.assertEquals(Long.valueOf(30), testBranch.get(0).getAs("max_snapshot_age_in_ms")); + assertThat(testBranch) + .hasSize(1) + .containsExactly( + RowFactory.create("testBranch", "BRANCH", currentSnapshotId, 10L, 20L, 30L)); + assertThat(testBranch.get(0).schema().fieldNames()) + .containsExactly( + "name", + "type", + "snapshot_id", + "max_reference_age_in_ms", + "min_snapshots_to_keep", + "max_snapshot_age_in_ms"); // Check tag entries in refs table List testTag = spark .sql("SELECT * FROM " + tableName + ".refs WHERE name = 'testTag' AND type='TAG'") .collectAsList(); - Assert.assertEquals("testTag", testTag.get(0).getAs("name")); - Assert.assertEquals("TAG", testTag.get(0).getAs("type")); - Assert.assertEquals(currentSnapshotId, testTag.get(0).getAs("snapshot_id")); - Assert.assertEquals(Long.valueOf(50), testTag.get(0).getAs("max_reference_age_in_ms")); + assertThat(testTag) + .hasSize(1) + .containsExactly(RowFactory.create("testTag", "TAG", currentSnapshotId, 50L, null, null)); + assertThat(testTag.get(0).schema().fieldNames()) + .containsExactly( + "name", + "type", + "snapshot_id", + "max_reference_age_in_ms", + "min_snapshots_to_keep", + "max_snapshot_age_in_ms"); // Check projection in refs table List testTagProjection = @@ -651,12 +663,12 @@ public void testSnapshotReferencesMetatable() throws Exception { + tableName + ".refs where type='TAG'") .collectAsList(); - Assert.assertEquals("testTag", testTagProjection.get(0).getAs("name")); - Assert.assertEquals("TAG", testTagProjection.get(0).getAs("type")); - Assert.assertEquals(currentSnapshotId, testTagProjection.get(0).getAs("snapshot_id")); - Assert.assertEquals( - Long.valueOf(50), testTagProjection.get(0).getAs("max_reference_age_in_ms")); - Assert.assertNull(testTagProjection.get(0).getAs("min_snapshots_to_keep")); + assertThat(testTagProjection) + .hasSize(1) + .containsExactly(RowFactory.create("testTag", "TAG", currentSnapshotId, 50L, null)); + assertThat(testTagProjection.get(0).schema().fieldNames()) + .containsExactly( + "name", "type", "snapshot_id", "max_reference_age_in_ms", "min_snapshots_to_keep"); List mainBranchProjection = spark @@ -665,21 +677,23 @@ public void testSnapshotReferencesMetatable() throws Exception { + tableName + ".refs WHERE name = 'main' AND type = 'BRANCH'") .collectAsList(); - Assert.assertEquals("main", mainBranchProjection.get(0).getAs("name")); - Assert.assertEquals("BRANCH", mainBranchProjection.get(0).getAs("type")); + assertThat(mainBranchProjection) + .hasSize(1) + .containsExactly(RowFactory.create("main", "BRANCH")); + assertThat(mainBranchProjection.get(0).schema().fieldNames()).containsExactly("name", "type"); List testBranchProjection = spark .sql( - "SELECT type, name, max_reference_age_in_ms, snapshot_id FROM " + "SELECT name, type, snapshot_id, max_reference_age_in_ms FROM " + tableName + ".refs WHERE name = 'testBranch' AND type = 'BRANCH'") .collectAsList(); - Assert.assertEquals("testBranch", testBranchProjection.get(0).getAs("name")); - Assert.assertEquals("BRANCH", testBranchProjection.get(0).getAs("type")); - Assert.assertEquals(currentSnapshotId, testBranchProjection.get(0).getAs("snapshot_id")); - Assert.assertEquals( - Long.valueOf(10), testBranchProjection.get(0).getAs("max_reference_age_in_ms")); + assertThat(testBranchProjection) + .hasSize(1) + .containsExactly(RowFactory.create("testBranch", "BRANCH", currentSnapshotId, 10L)); + assertThat(testBranchProjection.get(0).schema().fieldNames()) + .containsExactly("name", "type", "snapshot_id", "max_reference_age_in_ms"); } /** @@ -724,7 +738,7 @@ private boolean partitionMatch(Record file, String partValue) { return partValue.equals(partition.get(0).toString()); } - @Test + @TestTemplate public void metadataLogEntriesAfterReplacingTable() throws Exception { sql( "CREATE TABLE %s (id bigint, data string) " diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java index 00130b4d7e14..ce609450c097 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java @@ -20,9 +20,12 @@ import static org.apache.iceberg.RowLevelOperationMode.COPY_ON_WRITE; import static org.apache.iceberg.RowLevelOperationMode.MERGE_ON_READ; +import static org.assertj.core.api.Assertions.assertThat; import java.util.Map; import org.apache.commons.lang3.StringUtils; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; import org.apache.iceberg.RowLevelOperationMode; import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; @@ -31,13 +34,12 @@ import org.apache.iceberg.spark.SparkSQLProperties; import org.apache.spark.sql.execution.SparkPlan; import org.apache.spark.sql.internal.SQLConf; -import org.assertj.core.api.Assertions; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; -public class TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtensionsTestBase { +@ExtendWith(ParameterizedTestExtension.class) +public class TestStoragePartitionedJoinsInRowLevelOperations extends ExtensionsTestBase { private static final String OTHER_TABLE_NAME = "other_table"; @@ -68,7 +70,7 @@ public class TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens SparkSQLProperties.PRESERVE_DATA_GROUPING, "true"); - @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") public static Object[][] parameters() { return new Object[][] { { @@ -79,23 +81,18 @@ public static Object[][] parameters() { }; } - public TestStoragePartitionedJoinsInRowLevelOperations( - String catalogName, String implementation, Map config) { - super(catalogName, implementation, config); - } - - @After + @AfterEach public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); sql("DROP TABLE IF EXISTS %s", tableName(OTHER_TABLE_NAME)); } - @Test + @TestTemplate public void testCopyOnWriteDeleteWithoutShuffles() { checkDelete(COPY_ON_WRITE); } - @Test + @TestTemplate public void testMergeOnReadDeleteWithoutShuffles() { checkDelete(MERGE_ON_READ); } @@ -139,10 +136,10 @@ private void checkDelete(RowLevelOperationMode mode) { String planAsString = plan.toString(); if (mode == COPY_ON_WRITE) { int actualNumShuffles = StringUtils.countMatches(planAsString, "Exchange"); - Assert.assertEquals("Should be 1 shuffle with SPJ", 1, actualNumShuffles); - Assertions.assertThat(planAsString).contains("Exchange hashpartitioning(_file"); + assertThat(actualNumShuffles).as("Should be 1 shuffle with SPJ").isEqualTo(1); + assertThat(planAsString).contains("Exchange hashpartitioning(_file"); } else { - Assertions.assertThat(planAsString).doesNotContain("Exchange"); + assertThat(planAsString).doesNotContain("Exchange"); } }); @@ -158,12 +155,12 @@ private void checkDelete(RowLevelOperationMode mode) { sql("SELECT * FROM %s ORDER BY id, salary", tableName)); } - @Test + @TestTemplate public void testCopyOnWriteUpdateWithoutShuffles() { checkUpdate(COPY_ON_WRITE); } - @Test + @TestTemplate public void testMergeOnReadUpdateWithoutShuffles() { checkUpdate(MERGE_ON_READ); } @@ -207,10 +204,10 @@ private void checkUpdate(RowLevelOperationMode mode) { String planAsString = plan.toString(); if (mode == COPY_ON_WRITE) { int actualNumShuffles = StringUtils.countMatches(planAsString, "Exchange"); - Assert.assertEquals("Should be 1 shuffle with SPJ", 1, actualNumShuffles); - Assertions.assertThat(planAsString).contains("Exchange hashpartitioning(_file"); + assertThat(actualNumShuffles).as("Should be 1 shuffle with SPJ").isEqualTo(1); + assertThat(planAsString).contains("Exchange hashpartitioning(_file"); } else { - Assertions.assertThat(planAsString).doesNotContain("Exchange"); + assertThat(planAsString).doesNotContain("Exchange"); } }); @@ -227,22 +224,22 @@ private void checkUpdate(RowLevelOperationMode mode) { sql("SELECT * FROM %s ORDER BY id, salary", tableName)); } - @Test + @TestTemplate public void testCopyOnWriteMergeWithoutShuffles() { checkMerge(COPY_ON_WRITE, false /* with ON predicate */); } - @Test + @TestTemplate public void testCopyOnWriteMergeWithoutShufflesWithPredicate() { checkMerge(COPY_ON_WRITE, true /* with ON predicate */); } - @Test + @TestTemplate public void testMergeOnReadMergeWithoutShuffles() { checkMerge(MERGE_ON_READ, false /* with ON predicate */); } - @Test + @TestTemplate public void testMergeOnReadMergeWithoutShufflesWithPredicate() { checkMerge(MERGE_ON_READ, true /* with ON predicate */); } @@ -294,10 +291,10 @@ private void checkMerge(RowLevelOperationMode mode, boolean withPredicate) { String planAsString = plan.toString(); if (mode == COPY_ON_WRITE) { int actualNumShuffles = StringUtils.countMatches(planAsString, "Exchange"); - Assert.assertEquals("Should be 1 shuffle with SPJ", 1, actualNumShuffles); - Assertions.assertThat(planAsString).contains("Exchange hashpartitioning(_file"); + assertThat(actualNumShuffles).as("Should be 1 shuffle with SPJ").isEqualTo(1); + assertThat(planAsString).contains("Exchange hashpartitioning(_file"); } else { - Assertions.assertThat(planAsString).doesNotContain("Exchange"); + assertThat(planAsString).doesNotContain("Exchange"); } }); diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java index 7f2857cce0b9..f6102bab69b0 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java @@ -37,9 +37,11 @@ import static org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStrToHourOrdinal; import static org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStrToMonthOrdinal; import static org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStrToYearOrdinal; +import static org.assertj.core.api.Assertions.assertThat; import java.util.List; -import java.util.Map; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; import org.apache.iceberg.expressions.ExpressionUtil; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.PlanUtils; @@ -49,19 +51,15 @@ import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; -import org.assertj.core.api.Assertions; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runners.Parameterized; - -public class TestSystemFunctionPushDownDQL extends SparkExtensionsTestBase { - public TestSystemFunctionPushDownDQL( - String catalogName, String implementation, Map config) { - super(catalogName, implementation, config); - } +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ParameterizedTestExtension.class) +public class TestSystemFunctionPushDownDQL extends ExtensionsTestBase { - @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") public static Object[][] parameters() { return new Object[][] { { @@ -72,23 +70,24 @@ public static Object[][] parameters() { }; } - @Before + @BeforeEach public void before() { + super.before(); sql("USE %s", catalogName); } - @After + @AfterEach public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } - @Test + @TestTemplate public void testYearsFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testYearsFunction(false); } - @Test + @TestTemplate public void testYearsFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "years(ts)"); testYearsFunction(true); @@ -107,16 +106,16 @@ private void testYearsFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, equal(year("ts"), targetYears)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(5); + assertThat(actual).hasSize(5); } - @Test + @TestTemplate public void testMonthsFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testMonthsFunction(false); } - @Test + @TestTemplate public void testMonthsFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "months(ts)"); testMonthsFunction(true); @@ -135,16 +134,16 @@ private void testMonthsFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, greaterThan(month("ts"), targetMonths)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(5); + assertThat(actual).hasSize(5); } - @Test + @TestTemplate public void testDaysFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testDaysFunction(false); } - @Test + @TestTemplate public void testDaysFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "days(ts)"); testDaysFunction(true); @@ -165,16 +164,16 @@ private void testDaysFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, lessThan(day("ts"), targetDays)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(5); + assertThat(actual).hasSize(5); } - @Test + @TestTemplate public void testHoursFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testHoursFunction(false); } - @Test + @TestTemplate public void testHoursFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "hours(ts)"); testHoursFunction(true); @@ -193,16 +192,16 @@ private void testHoursFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, greaterThanOrEqual(hour("ts"), targetHours)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(8); + assertThat(actual).hasSize(8); } - @Test + @TestTemplate public void testBucketLongFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testBucketLongFunction(false); } - @Test + @TestTemplate public void testBucketLongFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "bucket(5, id)"); testBucketLongFunction(true); @@ -221,16 +220,16 @@ private void testBucketLongFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, lessThanOrEqual(bucket("id", 5), target)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(5); + assertThat(actual).hasSize(5); } - @Test + @TestTemplate public void testBucketStringFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testBucketStringFunction(false); } - @Test + @TestTemplate public void testBucketStringFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "bucket(5, data)"); testBucketStringFunction(true); @@ -249,16 +248,16 @@ private void testBucketStringFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, notEqual(bucket("data", 5), target)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(8); + assertThat(actual).hasSize(8); } - @Test + @TestTemplate public void testTruncateFunctionOnUnpartitionedTable() { createUnpartitionedTable(spark, tableName); testTruncateFunction(false); } - @Test + @TestTemplate public void testTruncateFunctionOnPartitionedTable() { createPartitionedTable(spark, tableName, "truncate(4, data)"); testTruncateFunction(true); @@ -278,7 +277,7 @@ private void testTruncateFunction(boolean partitioned) { checkPushedFilters(optimizedPlan, equal(truncate("data", 4), target)); List actual = rowsToJava(df.collectAsList()); - Assertions.assertThat(actual.size()).isEqualTo(5); + assertThat(actual).hasSize(5); } private void checkExpressions( @@ -286,18 +285,18 @@ private void checkExpressions( List staticInvokes = PlanUtils.collectSparkExpressions( optimizedPlan, expression -> expression instanceof StaticInvoke); - Assertions.assertThat(staticInvokes).isEmpty(); + assertThat(staticInvokes).isEmpty(); List applyExpressions = PlanUtils.collectSparkExpressions( optimizedPlan, expression -> expression instanceof ApplyFunctionExpression); if (partitioned) { - Assertions.assertThat(applyExpressions).isEmpty(); + assertThat(applyExpressions).isEmpty(); } else { - Assertions.assertThat(applyExpressions.size()).isEqualTo(1); + assertThat(applyExpressions).hasSize(1); ApplyFunctionExpression expression = (ApplyFunctionExpression) applyExpressions.get(0); - Assertions.assertThat(expression.name()).isEqualTo(expectedFunctionName); + assertThat(expression.name()).isEqualTo(expectedFunctionName); } } @@ -305,9 +304,9 @@ private void checkPushedFilters( LogicalPlan optimizedPlan, org.apache.iceberg.expressions.Expression expected) { List pushedFilters = PlanUtils.collectPushDownFilters(optimizedPlan); - Assertions.assertThat(pushedFilters.size()).isEqualTo(1); + assertThat(pushedFilters).hasSize(1); org.apache.iceberg.expressions.Expression actual = pushedFilters.get(0); - Assertions.assertThat(ExpressionUtil.equivalent(expected, actual, STRUCT, true)) + assertThat(ExpressionUtil.equivalent(expected, actual, STRUCT, true)) .as("Pushed filter should match") .isTrue(); } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java index ac02561a3c76..786d0bd1c660 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java @@ -23,11 +23,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.List; -import java.util.Map; import java.util.Random; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.iceberg.IcebergBuild; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -51,30 +52,32 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.catalyst.catalog.SessionCatalog; import org.assertj.core.api.InstanceOfAssertFactories; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; -public class TestViews extends SparkExtensionsTestBase { +@ExtendWith(ParameterizedTestExtension.class) +public class TestViews extends ExtensionsTestBase { private static final Namespace NAMESPACE = Namespace.of("default"); private final String tableName = "table"; - @Before + @BeforeEach public void before() { + super.before(); spark.conf().set("spark.sql.defaultCatalog", catalogName); sql("USE %s", catalogName); sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); sql("CREATE TABLE %s (id INT, data STRING)", tableName); } - @After + @AfterEach public void removeTable() { sql("USE %s", catalogName); sql("DROP TABLE IF EXISTS %s", tableName); } - @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") public static Object[][] parameters() { return new Object[][] { { @@ -85,11 +88,7 @@ public static Object[][] parameters() { }; } - public TestViews(String catalog, String implementation, Map properties) { - super(catalog, implementation, properties); - } - - @Test + @TestTemplate public void readFromView() throws NoSuchTableException { insertRows(10); String viewName = "simpleView"; @@ -115,7 +114,7 @@ public void readFromView() throws NoSuchTableException { .containsExactlyInAnyOrderElementsOf(expected); } - @Test + @TestTemplate public void readFromTrinoView() throws NoSuchTableException { insertRows(10); String viewName = "trinoView"; @@ -140,7 +139,7 @@ public void readFromTrinoView() throws NoSuchTableException { .containsExactlyInAnyOrderElementsOf(expected); } - @Test + @TestTemplate public void readFromMultipleViews() throws NoSuchTableException { insertRows(6); String viewName = "firstView"; @@ -173,7 +172,7 @@ public void readFromMultipleViews() throws NoSuchTableException { .containsExactlyInAnyOrder(row(4), row(5), row(6)); } - @Test + @TestTemplate public void readFromViewUsingNonExistingTable() throws NoSuchTableException { insertRows(10); String viewName = "viewWithNonExistingTable"; @@ -197,7 +196,7 @@ public void readFromViewUsingNonExistingTable() throws NoSuchTableException { catalogName, NAMESPACE)); } - @Test + @TestTemplate public void readFromViewUsingNonExistingTableColumn() throws NoSuchTableException { insertRows(10); String viewName = "viewWithNonExistingColumn"; @@ -219,7 +218,7 @@ public void readFromViewUsingNonExistingTableColumn() throws NoSuchTableExceptio "A column or function parameter with name `non_existing` cannot be resolved"); } - @Test + @TestTemplate public void readFromViewUsingInvalidSQL() throws NoSuchTableException { insertRows(10); String viewName = "viewWithInvalidSQL"; @@ -241,7 +240,7 @@ public void readFromViewUsingInvalidSQL() throws NoSuchTableException { String.format("The view `%s` cannot be displayed due to invalid view text", viewName)); } - @Test + @TestTemplate public void readFromViewWithStaleSchema() throws NoSuchTableException { insertRows(10); String viewName = "staleView"; @@ -267,7 +266,7 @@ public void readFromViewWithStaleSchema() throws NoSuchTableException { .hasMessageContaining("A column or function parameter with name `data` cannot be resolved"); } - @Test + @TestTemplate public void readFromViewHiddenByTempView() throws NoSuchTableException { insertRows(10); String viewName = "viewHiddenByTempView"; @@ -294,7 +293,7 @@ public void readFromViewHiddenByTempView() throws NoSuchTableException { .containsExactlyInAnyOrderElementsOf(expected); } - @Test + @TestTemplate public void readFromViewWithGlobalTempView() throws NoSuchTableException { insertRows(10); String viewName = "viewWithGlobalTempView"; @@ -324,7 +323,7 @@ public void readFromViewWithGlobalTempView() throws NoSuchTableException { IntStream.rangeClosed(6, 10).mapToObj(this::row).collect(Collectors.toList())); } - @Test + @TestTemplate public void readFromViewReferencingAnotherView() throws NoSuchTableException { insertRows(10); String firstView = "viewBeingReferencedInAnotherView"; @@ -355,7 +354,7 @@ public void readFromViewReferencingAnotherView() throws NoSuchTableException { .containsExactly(row(5)); } - @Test + @TestTemplate public void readFromViewReferencingTempView() throws NoSuchTableException { insertRows(10); String tempView = "tempViewBeingReferencedInAnotherView"; @@ -391,7 +390,7 @@ public void readFromViewReferencingTempView() throws NoSuchTableException { .hasMessageContaining("cannot be found"); } - @Test + @TestTemplate public void readFromViewReferencingAnotherViewHiddenByTempView() throws NoSuchTableException { insertRows(10); String innerViewName = "inner_view"; @@ -439,7 +438,7 @@ public void readFromViewReferencingAnotherViewHiddenByTempView() throws NoSuchTa .containsExactlyInAnyOrderElementsOf(expectedViewRows); } - @Test + @TestTemplate public void readFromViewReferencingGlobalTempView() throws NoSuchTableException { insertRows(10); String globalTempView = "globalTempViewBeingReferenced"; @@ -477,7 +476,7 @@ public void readFromViewReferencingGlobalTempView() throws NoSuchTableException .hasMessageContaining("cannot be found"); } - @Test + @TestTemplate public void readFromViewReferencingTempFunction() throws NoSuchTableException { insertRows(10); String viewName = viewName("viewReferencingTempFunction"); @@ -510,7 +509,7 @@ public void readFromViewReferencingTempFunction() throws NoSuchTableException { .hasMessageContaining("cannot be found"); } - @Test + @TestTemplate public void readFromViewWithCTE() throws NoSuchTableException { insertRows(10); String viewName = "viewWithCTE"; @@ -533,7 +532,7 @@ public void readFromViewWithCTE() throws NoSuchTableException { assertThat(sql("SELECT * FROM %s", viewName)).hasSize(1).containsExactly(row(10, 1L)); } - @Test + @TestTemplate public void rewriteFunctionIdentifier() { String viewName = "rewriteFunctionIdentifier"; String sql = "SELECT iceberg_version() AS version"; @@ -559,7 +558,7 @@ public void rewriteFunctionIdentifier() { .containsExactly(row(IcebergBuild.version())); } - @Test + @TestTemplate public void builtinFunctionIdentifierNotRewritten() { String viewName = "builtinFunctionIdentifierNotRewritten"; String sql = "SELECT trim(' abc ') AS result"; @@ -578,7 +577,7 @@ public void builtinFunctionIdentifierNotRewritten() { assertThat(sql("SELECT * FROM %s", viewName)).hasSize(1).containsExactly(row("abc")); } - @Test + @TestTemplate public void rewriteFunctionIdentifierWithNamespace() { String viewName = "rewriteFunctionIdentifierWithNamespace"; String sql = "SELECT system.bucket(100, 'a') AS bucket_result, 'a' AS value"; @@ -605,7 +604,7 @@ public void rewriteFunctionIdentifierWithNamespace() { .containsExactly(row(50, "a")); } - @Test + @TestTemplate public void fullFunctionIdentifier() { String viewName = "fullFunctionIdentifier"; String sql = @@ -629,7 +628,7 @@ public void fullFunctionIdentifier() { .containsExactly(row(50, "a")); } - @Test + @TestTemplate public void fullFunctionIdentifierNotRewrittenLoadFailure() { String viewName = "fullFunctionIdentifierNotRewrittenLoadFailure"; String sql = "SELECT spark_catalog.system.bucket(100, 'a') AS bucket_result, 'a' AS value"; @@ -674,7 +673,7 @@ private Catalog tableCatalog() { return Spark3Util.loadIcebergCatalog(spark, catalogName); } - @Test + @TestTemplate public void renameView() throws NoSuchTableException { insertRows(10); String viewName = viewName("originalView"); @@ -700,7 +699,7 @@ public void renameView() throws NoSuchTableException { .containsExactlyInAnyOrderElementsOf(expected); } - @Test + @TestTemplate public void renameViewHiddenByTempView() throws NoSuchTableException { insertRows(10); String viewName = viewName("originalView"); @@ -739,7 +738,7 @@ public void renameViewHiddenByTempView() throws NoSuchTableException { assertThat(viewCatalog.viewExists(TableIdentifier.of(NAMESPACE, renamedView))).isTrue(); } - @Test + @TestTemplate public void renameViewToDifferentTargetCatalog() { String viewName = viewName("originalView"); String renamedView = viewName("renamedView"); @@ -761,14 +760,14 @@ public void renameViewToDifferentTargetCatalog() { "Cannot move view between catalogs: from=spark_with_views and to=spark_catalog"); } - @Test + @TestTemplate public void renameNonExistingView() { assertThatThrownBy(() -> sql("ALTER VIEW non_existing RENAME TO target")) .isInstanceOf(AnalysisException.class) .hasMessageContaining("The table or view `non_existing` cannot be found"); } - @Test + @TestTemplate public void renameViewTargetAlreadyExistsAsView() { String viewName = viewName("renameViewSource"); String target = viewName("renameViewTarget"); @@ -798,7 +797,7 @@ public void renameViewTargetAlreadyExistsAsView() { String.format("Cannot create view default.%s because it already exists", target)); } - @Test + @TestTemplate public void renameViewTargetAlreadyExistsAsTable() { String viewName = viewName("renameViewSource"); String target = viewName("renameViewTarget"); @@ -821,7 +820,7 @@ public void renameViewTargetAlreadyExistsAsTable() { String.format("Cannot create view default.%s because it already exists", target)); } - @Test + @TestTemplate public void dropView() { String viewName = "viewToBeDropped"; String sql = String.format("SELECT id FROM %s", tableName); @@ -843,14 +842,14 @@ public void dropView() { assertThat(viewCatalog.viewExists(identifier)).isFalse(); } - @Test + @TestTemplate public void dropNonExistingView() { assertThatThrownBy(() -> sql("DROP VIEW non_existing")) .isInstanceOf(AnalysisException.class) .hasMessageContaining("The view %s.%s cannot be found", NAMESPACE, "non_existing"); } - @Test + @TestTemplate public void dropViewIfExists() { String viewName = "viewToBeDropped"; String sql = String.format("SELECT id FROM %s", tableName); @@ -875,7 +874,7 @@ public void dropViewIfExists() { } /** The purpose of this test is mainly to make sure that normal view deletion isn't messed up */ - @Test + @TestTemplate public void dropGlobalTempView() { String globalTempView = "globalViewToBeDropped"; sql("CREATE GLOBAL TEMPORARY VIEW %s AS SELECT id FROM %s", globalTempView, tableName); @@ -886,7 +885,7 @@ public void dropGlobalTempView() { } /** The purpose of this test is mainly to make sure that normal view deletion isn't messed up */ - @Test + @TestTemplate public void dropTempView() { String tempView = "tempViewToBeDropped"; sql("CREATE TEMPORARY VIEW %s AS SELECT id FROM %s", tempView, tableName); @@ -897,7 +896,7 @@ public void dropTempView() { } /** The purpose of this test is mainly to make sure that normal view deletion isn't messed up */ - @Test + @TestTemplate public void dropV1View() { String v1View = "v1ViewToBeDropped"; sql("USE spark_catalog"); @@ -928,7 +927,7 @@ private String viewName(String viewName) { return viewName + new Random().nextInt(1000000); } - @Test + @TestTemplate public void createViewIfNotExists() { String viewName = "viewThatAlreadyExists"; sql("CREATE VIEW %s AS SELECT id FROM %s", viewName, tableName); @@ -945,7 +944,7 @@ public void createViewIfNotExists() { () -> sql("CREATE VIEW IF NOT EXISTS %s AS SELECT id FROM %s", viewName, tableName)); } - @Test + @TestTemplate public void createOrReplaceView() throws NoSuchTableException { insertRows(6); String viewName = viewName("simpleView"); @@ -961,14 +960,14 @@ public void createOrReplaceView() throws NoSuchTableException { .containsExactlyInAnyOrder(row(4), row(5), row(6)); } - @Test + @TestTemplate public void createViewWithInvalidSQL() { assertThatThrownBy(() -> sql("CREATE VIEW simpleViewWithInvalidSQL AS invalid SQL")) .isInstanceOf(AnalysisException.class) .hasMessageContaining("Syntax error"); } - @Test + @TestTemplate public void createViewReferencingTempView() throws NoSuchTableException { insertRows(10); String tempView = "temporaryViewBeingReferencedInAnotherView"; @@ -987,7 +986,7 @@ public void createViewReferencingTempView() throws NoSuchTableException { .hasMessageContaining(tempView); } - @Test + @TestTemplate public void createViewReferencingGlobalTempView() throws NoSuchTableException { insertRows(10); String globalTempView = "globalTemporaryViewBeingReferenced"; @@ -1011,7 +1010,7 @@ public void createViewReferencingGlobalTempView() throws NoSuchTableException { .hasMessageContaining(String.format("%s.%s", "global_temp", globalTempView)); } - @Test + @TestTemplate public void createViewReferencingTempFunction() { String viewName = viewName("viewReferencingTemporaryFunction"); String functionName = "test_avg_func"; @@ -1030,7 +1029,7 @@ public void createViewReferencingTempFunction() { .hasMessageContaining(functionName); } - @Test + @TestTemplate public void createViewReferencingQualifiedTempFunction() { String viewName = viewName("viewReferencingTemporaryFunction"); String functionName = "test_avg_func_qualified"; @@ -1061,7 +1060,7 @@ public void createViewReferencingQualifiedTempFunction() { .hasMessageContaining(String.format("`%s`.`%s`", NAMESPACE, functionName)); } - @Test + @TestTemplate public void createViewUsingNonExistingTable() { assertThatThrownBy( () -> sql("CREATE VIEW viewWithNonExistingTable AS SELECT id FROM non_existing")) @@ -1069,7 +1068,7 @@ public void createViewUsingNonExistingTable() { .hasMessageContaining("The table or view `non_existing` cannot be found"); } - @Test + @TestTemplate public void createViewWithMismatchedColumnCounts() { String viewName = "viewWithMismatchedColumnCounts"; @@ -1092,7 +1091,7 @@ public void createViewWithMismatchedColumnCounts() { .hasMessageContaining("Data columns: id, data"); } - @Test + @TestTemplate public void createViewWithColumnAliases() throws NoSuchTableException { insertRows(6); String viewName = "viewWithColumnAliases"; @@ -1128,7 +1127,7 @@ public void createViewWithColumnAliases() throws NoSuchTableException { .containsExactlyInAnyOrder(row(1), row(2), row(3)); } - @Test + @TestTemplate public void createViewWithDuplicateColumnNames() { assertThatThrownBy( () -> @@ -1139,7 +1138,7 @@ public void createViewWithDuplicateColumnNames() { .hasMessageContaining("The column `new_id` already exists"); } - @Test + @TestTemplate public void createViewWithDuplicateQueryColumnNames() throws NoSuchTableException { insertRows(3); String viewName = "viewWithDuplicateQueryColumnNames"; @@ -1157,7 +1156,7 @@ public void createViewWithDuplicateQueryColumnNames() throws NoSuchTableExceptio .containsExactlyInAnyOrder(row(1, 1), row(2, 2), row(3, 3)); } - @Test + @TestTemplate public void createViewWithCTE() throws NoSuchTableException { insertRows(10); String viewName = "simpleViewWithCTE"; @@ -1172,7 +1171,7 @@ public void createViewWithCTE() throws NoSuchTableException { assertThat(sql("SELECT * FROM %s", viewName)).hasSize(1).containsExactly(row(10, 1L)); } - @Test + @TestTemplate public void createViewWithConflictingNamesForCTEAndTempView() throws NoSuchTableException { insertRows(10); String viewName = "viewWithConflictingNamesForCTEAndTempView"; @@ -1191,7 +1190,7 @@ public void createViewWithConflictingNamesForCTEAndTempView() throws NoSuchTable assertThat(sql("SELECT * FROM %s", viewName)).hasSize(1).containsExactly(row(10, 1L)); } - @Test + @TestTemplate public void createViewWithCTEReferencingTempView() { String viewName = "viewWithCTEReferencingTempView"; String tempViewInCTE = "tempViewInCTE"; @@ -1211,7 +1210,7 @@ public void createViewWithCTEReferencingTempView() { .hasMessageContaining(tempViewInCTE); } - @Test + @TestTemplate public void createViewWithCTEReferencingTempFunction() { String viewName = "viewWithCTEReferencingTempFunction"; String functionName = "avg_function_in_cte"; @@ -1233,7 +1232,7 @@ public void createViewWithCTEReferencingTempFunction() { .hasMessageContaining(functionName); } - @Test + @TestTemplate public void createViewWithNonExistingQueryColumn() { assertThatThrownBy( () -> @@ -1245,7 +1244,7 @@ public void createViewWithNonExistingQueryColumn() { "A column or function parameter with name `non_existing` cannot be resolved"); } - @Test + @TestTemplate public void createViewWithSubqueryExpressionUsingTempView() { String viewName = "viewWithSubqueryExpression"; String tempView = "simpleTempView"; @@ -1262,7 +1261,7 @@ public void createViewWithSubqueryExpressionUsingTempView() { .hasMessageContaining(tempView); } - @Test + @TestTemplate public void createViewWithSubqueryExpressionUsingGlobalTempView() { String viewName = "simpleViewWithSubqueryExpression"; String globalTempView = "simpleGlobalTempView"; @@ -1283,7 +1282,7 @@ public void createViewWithSubqueryExpressionUsingGlobalTempView() { .hasMessageContaining(String.format("%s.%s", "global_temp", globalTempView)); } - @Test + @TestTemplate public void createViewWithSubqueryExpressionUsingTempFunction() { String viewName = viewName("viewWithSubqueryExpression"); String functionName = "avg_function_in_subquery"; @@ -1304,7 +1303,7 @@ public void createViewWithSubqueryExpressionUsingTempFunction() { .hasMessageContaining(functionName); } - @Test + @TestTemplate public void createViewWithSubqueryExpressionInFilterThatIsRewritten() throws NoSuchTableException { insertRows(5); @@ -1329,7 +1328,7 @@ public void createViewWithSubqueryExpressionInFilterThatIsRewritten() .containsExactly(row(5)); } - @Test + @TestTemplate public void createViewWithSubqueryExpressionInQueryThatIsRewritten() throws NoSuchTableException { insertRows(3); String viewName = viewName("viewWithSubqueryExpression"); @@ -1354,7 +1353,7 @@ public void createViewWithSubqueryExpressionInQueryThatIsRewritten() throws NoSu .containsExactly(row(3), row(3), row(3)); } - @Test + @TestTemplate public void describeView() { String viewName = "describeView"; @@ -1363,7 +1362,7 @@ public void describeView() { .containsExactly(row("id", "int", ""), row("data", "string", "")); } - @Test + @TestTemplate public void describeExtendedView() { String viewName = "describeExtendedView"; String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName); @@ -1388,7 +1387,7 @@ public void describeExtendedView() { "")); } - @Test + @TestTemplate public void showViewProperties() { String viewName = "showViewProps"; @@ -1399,7 +1398,7 @@ public void showViewProperties() { .contains(row("key1", "val1"), row("key2", "val2")); } - @Test + @TestTemplate public void showViewPropertiesByKey() { String viewName = "showViewPropsByKey"; @@ -1418,7 +1417,7 @@ public void showViewPropertiesByKey() { catalogName, NAMESPACE, viewName))); } - @Test + @TestTemplate public void showViews() throws NoSuchTableException { insertRows(6); String sql = String.format("SELECT * from %s", tableName); @@ -1466,7 +1465,7 @@ public void showViews() throws NoSuchTableException { row("global_temp", "globalviewforlisting", true), tempView); } - @Test + @TestTemplate public void showViewsWithCurrentNamespace() { String namespaceOne = "show_views_ns1"; String namespaceTwo = "show_views_ns2"; @@ -1497,7 +1496,7 @@ public void showViewsWithCurrentNamespace() { assertThat(sql("SHOW VIEWS LIKE 'viewTwo*'")).contains(v2).doesNotContain(v1); } - @Test + @TestTemplate public void showCreateSimpleView() { String viewName = "showCreateSimpleView"; String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName); @@ -1518,7 +1517,7 @@ public void showCreateSimpleView() { assertThat(sql("SHOW CREATE TABLE %s", viewName)).containsExactly(row(expected)); } - @Test + @TestTemplate public void showCreateComplexView() { String viewName = "showCreateComplexView"; String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName); @@ -1545,7 +1544,7 @@ public void showCreateComplexView() { assertThat(sql("SHOW CREATE TABLE %s", viewName)).containsExactly(row(expected)); } - @Test + @TestTemplate public void alterViewSetProperties() { String viewName = "viewWithSetProperties"; @@ -1567,7 +1566,7 @@ public void alterViewSetProperties() { .containsEntry("comment", "view comment"); } - @Test + @TestTemplate public void alterViewSetReservedProperties() { String viewName = "viewWithSetReservedProperties"; @@ -1598,7 +1597,7 @@ public void alterViewSetReservedProperties() { .hasMessageContaining("Cannot set reserved property: 'spark.query-column-names'"); } - @Test + @TestTemplate public void alterViewUnsetProperties() { String viewName = "viewWithUnsetProperties"; sql("CREATE VIEW %s AS SELECT id FROM %s WHERE id <= 3", viewName, tableName); @@ -1619,7 +1618,7 @@ public void alterViewUnsetProperties() { .containsEntry("comment", "view comment"); } - @Test + @TestTemplate public void alterViewUnsetUnknownProperty() { String viewName = "viewWithUnsetUnknownProp"; sql("CREATE VIEW %s AS SELECT id FROM %s WHERE id <= 3", viewName, tableName); @@ -1633,7 +1632,7 @@ public void alterViewUnsetUnknownProperty() { () -> sql("ALTER VIEW %s UNSET TBLPROPERTIES IF EXISTS ('unknown-key')", viewName)); } - @Test + @TestTemplate public void alterViewUnsetReservedProperties() { String viewName = "viewWithUnsetReservedProperties"; @@ -1669,7 +1668,7 @@ public void alterViewUnsetReservedProperties() { .hasMessageContaining("Cannot unset reserved property: 'spark.query-column-names'"); } - @Test + @TestTemplate public void createOrReplaceViewWithColumnAliases() throws NoSuchTableException { insertRows(6); String viewName = viewName("viewWithColumnAliases"); @@ -1715,7 +1714,7 @@ public void createOrReplaceViewWithColumnAliases() throws NoSuchTableException { assertThat(second.doc()).isEqualTo("new ID"); } - @Test + @TestTemplate public void alterViewIsNotSupported() throws NoSuchTableException { insertRows(6); String viewName = "alteredView"; @@ -1733,7 +1732,7 @@ public void alterViewIsNotSupported() throws NoSuchTableException { "ALTER VIEW AS is not supported. Use CREATE OR REPLACE VIEW instead"); } - @Test + @TestTemplate public void createOrReplaceViewKeepsViewHistory() { String viewName = viewName("viewWithHistoryAfterReplace"); String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName); @@ -1772,7 +1771,7 @@ public void createOrReplaceViewKeepsViewHistory() { .asStruct()); } - @Test + @TestTemplate public void replacingTrinoViewShouldFail() { String viewName = viewName("trinoView"); String sql = String.format("SELECT id FROM %s", tableName); @@ -1795,7 +1794,7 @@ public void replacingTrinoViewShouldFail() { + "New dialects: [spark]"); } - @Test + @TestTemplate public void replacingTrinoAndSparkViewShouldFail() { String viewName = viewName("trinoAndSparkView"); String sql = String.format("SELECT id FROM %s", tableName); @@ -1819,7 +1818,7 @@ public void replacingTrinoAndSparkViewShouldFail() { + "New dialects: [spark]"); } - @Test + @TestTemplate public void replacingViewWithDialectDropAllowed() { String viewName = viewName("trinoView"); String sql = String.format("SELECT id FROM %s", tableName); diff --git a/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java b/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java index 25d7e7471588..207fca3cc216 100644 --- a/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java +++ b/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java @@ -19,22 +19,19 @@ package org.apache.iceberg.spark; import java.io.IOException; -import java.util.Map; +import java.nio.file.Files; +import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.spark.extensions.SparkExtensionsTestBase; -import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class SmokeTest extends SparkExtensionsTestBase { - - public SmokeTest(String catalogName, String implementation, Map config) { - super(catalogName, implementation, config); - } - - @Before +import org.apache.iceberg.spark.extensions.ExtensionsTestBase; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ParameterizedTestExtension.class) +public class SmokeTest extends ExtensionsTestBase { + @AfterEach public void dropTable() { sql("DROP TABLE IF EXISTS %s", tableName); } @@ -42,30 +39,32 @@ public void dropTable() { // Run through our Doc's Getting Started Example // TODO Update doc example so that it can actually be run, modifications were required for this // test suite to run - @Test + @TestTemplate public void testGettingStarted() throws IOException { // Creating a table sql("CREATE TABLE %s (id bigint, data string) USING iceberg", tableName); // Writing sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); - Assert.assertEquals( - "Should have inserted 3 rows", 3L, scalarSql("SELECT COUNT(*) FROM %s", tableName)); + Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s", tableName)) + .as("Should have inserted 3 rows") + .isEqualTo(3L); sql("DROP TABLE IF EXISTS source PURGE"); sql( "CREATE TABLE source (id bigint, data string) USING parquet LOCATION '%s'", - temp.newFolder()); + Files.createTempDirectory(temp, "junit")); sql("INSERT INTO source VALUES (10, 'd'), (11, 'ee')"); sql("INSERT INTO %s SELECT id, data FROM source WHERE length(data) = 1", tableName); - Assert.assertEquals( - "Table should now have 4 rows", 4L, scalarSql("SELECT COUNT(*) FROM %s", tableName)); + Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s", tableName)) + .as("Table should now have 4 rows") + .isEqualTo(4L); sql("DROP TABLE IF EXISTS updates PURGE"); sql( "CREATE TABLE updates (id bigint, data string) USING parquet LOCATION '%s'", - temp.newFolder()); + Files.createTempDirectory(temp, "junit")); sql("INSERT INTO updates VALUES (1, 'x'), (2, 'x'), (4, 'z')"); sql( @@ -73,31 +72,31 @@ public void testGettingStarted() throws IOException { + "WHEN MATCHED THEN UPDATE SET t.data = u.data\n" + "WHEN NOT MATCHED THEN INSERT *", tableName); - Assert.assertEquals( - "Table should now have 5 rows", 5L, scalarSql("SELECT COUNT(*) FROM %s", tableName)); - Assert.assertEquals( - "Record 1 should now have data x", - "x", - scalarSql("SELECT data FROM %s WHERE id = 1", tableName)); + Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s", tableName)) + .as("Table should now have 5 rows") + .isEqualTo(5L); + Assertions.assertThat(scalarSql("SELECT data FROM %s WHERE id = 1", tableName)) + .as("Record 1 should now have data x") + .isEqualTo("x"); // Reading - Assert.assertEquals( - "There should be 2 records with data x", - 2L, - scalarSql("SELECT count(1) as count FROM %s WHERE data = 'x' GROUP BY data ", tableName)); + Assertions.assertThat( + scalarSql( + "SELECT count(1) as count FROM %s WHERE data = 'x' GROUP BY data ", tableName)) + .as("There should be 2 records with data x") + .isEqualTo(2L); // Not supported because of Spark limitation if (!catalogName.equals("spark_catalog")) { - Assert.assertEquals( - "There should be 3 snapshots", - 3L, - scalarSql("SELECT COUNT(*) FROM %s.snapshots", tableName)); + Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s.snapshots", tableName)) + .as("There should be 3 snapshots") + .isEqualTo(3L); } } // From Spark DDL Docs section - @Test - public void testAlterTable() throws NoSuchTableException { + @TestTemplate + public void testAlterTable() { sql( "CREATE TABLE %s (category int, id bigint, data string, ts timestamp) USING iceberg", tableName); @@ -108,7 +107,9 @@ public void testAlterTable() throws NoSuchTableException { sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName); sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, category) AS shard", tableName); table = getTable(); - Assert.assertEquals("Table should have 4 partition fields", 4, table.spec().fields().size()); + Assertions.assertThat(table.spec().fields()) + .as("Table should have 4 partition fields") + .hasSize(4); // Drop Examples sql("ALTER TABLE %s DROP PARTITION FIELD bucket(16, id)", tableName); @@ -117,17 +118,21 @@ public void testAlterTable() throws NoSuchTableException { sql("ALTER TABLE %s DROP PARTITION FIELD shard", tableName); table = getTable(); - Assert.assertTrue("Table should be unpartitioned", table.spec().isUnpartitioned()); + Assertions.assertThat(table.spec().isUnpartitioned()) + .as("Table should be unpartitioned") + .isTrue(); // Sort order examples sql("ALTER TABLE %s WRITE ORDERED BY category, id", tableName); sql("ALTER TABLE %s WRITE ORDERED BY category ASC, id DESC", tableName); sql("ALTER TABLE %s WRITE ORDERED BY category ASC NULLS LAST, id DESC NULLS FIRST", tableName); table = getTable(); - Assert.assertEquals("Table should be sorted on 2 fields", 2, table.sortOrder().fields().size()); + Assertions.assertThat(table.sortOrder().fields()) + .as("Table should be sorted on 2 fields") + .hasSize(2); } - @Test + @TestTemplate public void testCreateTable() { sql("DROP TABLE IF EXISTS %s", tableName("first")); sql("DROP TABLE IF EXISTS %s", tableName("second")); @@ -150,7 +155,9 @@ public void testCreateTable() { + "PARTITIONED BY (category)", tableName("second")); Table second = getTable("second"); - Assert.assertEquals("Should be partitioned on 1 column", 1, second.spec().fields().size()); + Assertions.assertThat(second.spec().fields()) + .as("Should be partitioned on 1 column") + .hasSize(1); sql( "CREATE TABLE %s (\n" @@ -162,7 +169,9 @@ public void testCreateTable() { + "PARTITIONED BY (bucket(16, id), days(ts), category)", tableName("third")); Table third = getTable("third"); - Assert.assertEquals("Should be partitioned on 3 columns", 3, third.spec().fields().size()); + Assertions.assertThat(third.spec().fields()) + .as("Should be partitioned on 3 columns") + .hasSize(3); } private Table getTable(String name) {