diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileFormat.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileFormat.java index 26f74b65ac2e..14a42180a946 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileFormat.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileFormat.java @@ -39,6 +39,7 @@ public static IcebergFileFormat fromIceberg(FileFormat format) case AVRO -> AVRO; // Not used as a data file format case METADATA -> throw new IllegalArgumentException("Unexpected METADATA file format"); + case PUFFIN -> throw new IllegalArgumentException("Unexpected PUFFIN file format"); }; } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 0f2e7487883e..6a1eb954d99f 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -1474,7 +1474,7 @@ public Optional finishInsert( INCREMENTAL_UPDATE, collectedStatistics); transaction.updateStatistics() - .setStatistics(newSnapshotId, statisticsFile) + .setStatistics(statisticsFile) .commit(); commitTransaction(transaction, "update statistics on insert"); @@ -1986,7 +1986,7 @@ private void finishOptimize(ConnectorSession session, IcebergTableExecuteHandle StatisticsFile newStatsFile = tableStatisticsWriter.rewriteStatisticsFile(session, reloadedTable, newSnapshotId); transaction.updateStatistics() - .setStatistics(newSnapshotId, newStatsFile) + .setStatistics(newStatsFile) .commit(); commitTransaction(transaction, "update statistics after optimize"); } @@ -2931,7 +2931,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH REPLACE, collectedStatistics); transaction.updateStatistics() - .setStatistics(snapshotId, statisticsFile) + .setStatistics(statisticsFile) .commit(); commitTransaction(transaction, "statistics collection"); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/PartitionData.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/PartitionData.java index 3b715b9e5efa..a8806d566f28 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/PartitionData.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/PartitionData.java @@ -168,6 +168,9 @@ public static Object getValue(JsonNode partitionValue, Type type) createDecimalType(decimalType.precision(), decimalType.scale())); // TODO https://github.com/trinodb/trino/issues/19753 Support Iceberg timestamp types with nanosecond precision case TIMESTAMP_NANO: + // TODO https://github.com/trinodb/trino/issues/24538 Support variant type + case VARIANT: + case UNKNOWN: case LIST: case MAP: case STRUCT: diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TypeConverter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TypeConverter.java index 7216a7463ba9..05b8cda95964 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TypeConverter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TypeConverter.java @@ -105,6 +105,11 @@ public static Type toTrinoType(org.apache.iceberg.types.Type type, TypeManager t return RowType.from(fields.stream() .map(field -> new RowType.Field(Optional.of(field.name()), toTrinoType(field.type(), typeManager))) .collect(toImmutableList())); + case VARIANT: + // TODO https://github.com/trinodb/trino/issues/24538 Support variant type + break; + case UNKNOWN: + break; } throw new UnsupportedOperationException(format("Cannot convert from Iceberg type '%s' (%s) to Trino type", type, type.typeId())); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/HiveSchemaUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/HiveSchemaUtil.java index 9a97255871fc..89c97d7dc360 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/HiveSchemaUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/HiveSchemaUtil.java @@ -47,6 +47,9 @@ private static String convertToTypeString(Type type) case TIMESTAMP_NANO -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: TIMESTAMP_NANO"); case FIXED, BINARY -> "binary"; case DECIMAL -> "decimal(%s,%s)".formatted(((DecimalType) type).precision(), ((DecimalType) type).scale()); + case UNKNOWN -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: UNKNOWN"); + // TODO https://github.com/trinodb/trino/issues/24538 Support variant type + case VARIANT -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: VARIANT"); case LIST -> "array<%s>".formatted(convert(type.asListType().elementType())); case MAP -> "map<%s,%s>".formatted(convert(type.asMapType().keyType()), convert(type.asMapType().valueType())); case STRUCT -> "struct<%s>".formatted(type.asStructType().fields().stream() diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/OrcTypeConverter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/OrcTypeConverter.java index 413adc95ab2b..d1f1250372d5 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/OrcTypeConverter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/OrcTypeConverter.java @@ -85,6 +85,8 @@ private static List toOrcType(int nextFieldTypeIndex, Type type, Map throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: VARIANT"); + case UNKNOWN -> throw new TrinoException(NOT_SUPPORTED, "Unsupported Iceberg type: UNKNOWN"); case STRUCT -> toOrcStructType(nextFieldTypeIndex, (StructType) type, attributes); case LIST -> toOrcListType(nextFieldTypeIndex, (ListType) type, attributes); case MAP -> toOrcMapType(nextFieldTypeIndex, (MapType) type, attributes); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java index 776dbdd379ec..aaa5af59e926 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java @@ -172,7 +172,7 @@ public void testAnalyzeWithSchemaEvolution() double infoDataSize = (double) computeActual("SHOW STATS FOR " + tableName).getMaterializedRows().stream() .filter(row -> "info".equals(row.getField(0))) .collect(onlyElement()).getField(1); - assertThat(infoDataSize).isBetween(2000.0, 4000.0); + assertThat(infoDataSize).isBetween(2000.0, 5000.0); assertQuery( "SHOW STATS FOR " + tableName, """ diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java index b3a69a3a9cb8..4fe577411324 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java @@ -49,6 +49,7 @@ @Isolated // TODO remove @TestInstance(PER_CLASS) +@Disabled("https://github.com/trinodb/trino/issues/24945 Reenable after Polaris supports Iceberg versions >= 1.8.0") final class TestIcebergPolarisCatalogConnectorSmokeTest extends BaseIcebergConnectorSmokeTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java index f984aa7deef7..6d4a55cbbf13 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergUnityRestCatalogConnectorSmokeTest.java @@ -21,6 +21,7 @@ import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; import io.trino.testing.TestingConnectorBehavior; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -34,6 +35,7 @@ import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThatThrownBy; +@Disabled("https://github.com/trinodb/trino/issues/24945 Reenable after Unity supports Iceberg versions >= 1.8.0") final class TestIcebergUnityRestCatalogConnectorSmokeTest extends BaseIcebergConnectorSmokeTest { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java index 861f0efb97a6..afb5ec335a6b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java @@ -91,7 +91,8 @@ protected QueryRunner createQueryRunner() throws Exception { Network network = Network.newNetwork(); - minio = closeAfterClass(Minio.builder().withNetwork(network).build()); + // Use new MinIO version because Iceberg enables strong integrity checks + minio = closeAfterClass(Minio.builder().withImage("minio/minio:RELEASE.2025-01-20T14-49-07Z").withNetwork(network).build()); minio.start(); minio.createBucket(bucketName); diff --git a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/rest/RestCatalogServlet.java b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/rest/RestCatalogServlet.java index 0ccb550e88b2..e50c3fb58709 100644 --- a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/rest/RestCatalogServlet.java +++ b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/rest/RestCatalogServlet.java @@ -22,7 +22,7 @@ import org.apache.iceberg.exceptions.RESTException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.io.CharStreams; -import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod; +import org.apache.iceberg.rest.HTTPRequest.HTTPMethod; import org.apache.iceberg.rest.RESTCatalogAdapter.Route; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.iceberg.util.Pair; @@ -98,15 +98,19 @@ protected void execute(ServletRequestContext context, HttpServletResponse respon return; } + HTTPRequest request = restCatalogAdapter.buildRequest( + context.method(), + context.path(), + context.queryParams(), + context.headers(), + context.body()); + try { Object responseBody = restCatalogAdapter.execute( - context.method(), - context.path(), - context.queryParams(), - context.body(), + request, context.route().responseClass(), - context.headers(), - handle(response)); + handle(response), + _ -> {}); if (responseBody != null) { RESTObjectMapper.mapper().writeValue(response.getWriter(), responseBody); diff --git a/pom.xml b/pom.xml index 7c55d0fa94ad..034981a1d4ac 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ 1.12.780 4.17.0 7.7.1 - 108 + 109 1.22 11.3.1 1.15.1 @@ -198,7 +198,7 @@ 4.5.4 1.46.1 5.3.3 - 1.7.1 + 1.8.0 5.16.0 2.13.1 0.12.6 diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIceberg.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIceberg.java index 1064b3a9f6a8..8ba7188d5219 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIceberg.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIceberg.java @@ -32,6 +32,7 @@ import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; import static io.trino.tests.product.launcher.env.EnvironmentDefaults.HADOOP_BASE_IMAGE; import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_HADOOP_INIT_D; import static java.util.Objects.requireNonNull; @@ -48,6 +49,7 @@ public class EnvSinglenodeSparkIceberg private final DockerFiles dockerFiles; private final PortBinder portBinder; private final String hadoopImagesVersion; + private final DockerFiles.ResourceProvider configDir; @Inject public EnvSinglenodeSparkIceberg(Standard standard, Hadoop hadoop, DockerFiles dockerFiles, EnvironmentConfig config, PortBinder portBinder) @@ -56,6 +58,7 @@ public EnvSinglenodeSparkIceberg(Standard standard, Hadoop hadoop, DockerFiles d this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); this.portBinder = requireNonNull(portBinder, "portBinder is null"); this.hadoopImagesVersion = config.getHadoopImagesVersion(); + this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-spark-iceberg"); } @Override @@ -76,6 +79,8 @@ public void extendEnvironment(Environment.Builder builder) builder.configureContainer(TESTS, dockerContainer -> dockerContainer // Binding instead of copying for avoiding OutOfMemoryError https://github.com/testcontainers/testcontainers-java/issues/2863 .withFileSystemBind(HIVE_JDBC_PROVIDER.getParent(), "/docker/jdbc", BindMode.READ_ONLY)); + + configureTempto(builder, configDir); } @SuppressWarnings("resource") diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergJdbcCatalog.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergJdbcCatalog.java index dc206847a339..0a1e3f9b55d4 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergJdbcCatalog.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergJdbcCatalog.java @@ -32,6 +32,7 @@ import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; import static io.trino.tests.product.launcher.env.EnvironmentDefaults.HADOOP_BASE_IMAGE; import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_HADOOP_INIT_D; import static java.util.Objects.requireNonNull; @@ -50,6 +51,7 @@ public class EnvSinglenodeSparkIcebergJdbcCatalog private final DockerFiles dockerFiles; private final PortBinder portBinder; private final String imagesVersion; + private final DockerFiles.ResourceProvider configDir; @Inject public EnvSinglenodeSparkIcebergJdbcCatalog(Standard standard, Hadoop hadoop, DockerFiles dockerFiles, EnvironmentConfig config, PortBinder portBinder) @@ -58,6 +60,7 @@ public EnvSinglenodeSparkIcebergJdbcCatalog(Standard standard, Hadoop hadoop, Do this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); this.portBinder = requireNonNull(portBinder, "portBinder is null"); this.imagesVersion = requireNonNull(config, "config is null").getImagesVersion(); + this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-spark-iceberg-jdbc-catalog"); } @Override @@ -80,6 +83,8 @@ public void extendEnvironment(Environment.Builder builder) builder.configureContainer(TESTS, dockerContainer -> dockerContainer // Binding instead of copying for avoiding OutOfMemoryError https://github.com/testcontainers/testcontainers-java/issues/2863 .withFileSystemBind(HIVE_JDBC_PROVIDER.getParent(), "/docker/jdbc", BindMode.READ_ONLY)); + + configureTempto(builder, configDir); } @SuppressWarnings("resource") diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java index 1bc328fd18ac..c691a48af2b4 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java @@ -32,6 +32,7 @@ import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; import static java.util.Objects.requireNonNull; import static org.testcontainers.utility.MountableFile.forHostPath; @@ -49,6 +50,7 @@ public class EnvSinglenodeSparkIcebergNessie private final DockerFiles dockerFiles; private final PortBinder portBinder; private final String hadoopImagesVersion; + private final DockerFiles.ResourceProvider configDir; @Inject public EnvSinglenodeSparkIcebergNessie(Standard standard, Hadoop hadoop, DockerFiles dockerFiles, EnvironmentConfig config, PortBinder portBinder) @@ -57,6 +59,7 @@ public EnvSinglenodeSparkIcebergNessie(Standard standard, Hadoop hadoop, DockerF this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); this.portBinder = requireNonNull(portBinder, "portBinder is null"); this.hadoopImagesVersion = requireNonNull(config, "config is null").getHadoopImagesVersion(); + this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-spark-iceberg-nessie"); } @Override @@ -70,6 +73,8 @@ public void extendEnvironment(Environment.Builder builder) builder.configureContainer(TESTS, dockerContainer -> dockerContainer // Binding instead of copying for avoiding OutOfMemoryError https://github.com/testcontainers/testcontainers-java/issues/2863 .withFileSystemBind(HIVE_JDBC_PROVIDER.getParent(), "/docker/jdbc", BindMode.READ_ONLY)); + + configureTempto(builder, configDir); } @SuppressWarnings("resource") diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergRest.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergRest.java index 4854e212dfde..0d6418bb29e9 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergRest.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergRest.java @@ -21,6 +21,7 @@ import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentProvider; import io.trino.tests.product.launcher.env.common.Hadoop; +import io.trino.tests.product.launcher.env.common.Minio; import io.trino.tests.product.launcher.env.common.Standard; import io.trino.tests.product.launcher.env.common.TestsEnvironment; import io.trino.tests.product.launcher.testcontainers.PortBinder; @@ -28,10 +29,20 @@ import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; +import static io.trino.tests.product.launcher.env.common.Minio.MINIO_CONTAINER_NAME; import static java.util.Objects.requireNonNull; import static org.testcontainers.utility.MountableFile.forHostPath; @@ -48,20 +59,25 @@ public class EnvSinglenodeSparkIcebergRest private static final int REST_SERVER_PORT = 8181; private static final String SPARK_CONTAINER_NAME = "spark"; private static final String REST_CONTAINER_NAME = "iceberg-with-rest"; - private static final String REST_SERVER_IMAGE = "tabulario/iceberg-rest:1.5.0"; - private static final String CATALOG_WAREHOUSE = "hdfs://hadoop-master:9000/user/hive/warehouse"; + // TODO Pin version once Iceberg community releases it + private static final String REST_SERVER_IMAGE = "apache/iceberg-rest-fixture:latest"; + private static final String CATALOG_WAREHOUSE = "s3://test-bucket/default"; + private static final String S3_BUCKET_NAME = "test-bucket"; private final DockerFiles dockerFiles; private final PortBinder portBinder; private final String hadoopImagesVersion; + private final DockerFiles.ResourceProvider configDir; @Inject - public EnvSinglenodeSparkIcebergRest(Standard standard, Hadoop hadoop, DockerFiles dockerFiles, EnvironmentConfig config, PortBinder portBinder) + public EnvSinglenodeSparkIcebergRest(Standard standard, Hadoop hadoop, Minio minio, DockerFiles dockerFiles, EnvironmentConfig config, PortBinder portBinder) { - super(ImmutableList.of(standard, hadoop)); + // TODO Remove Hadoop when replace Hadoop with MinIO in all Iceberg product tests + super(ImmutableList.of(standard, hadoop, minio)); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); this.portBinder = requireNonNull(portBinder, "portBinder is null"); this.hadoopImagesVersion = requireNonNull(config, "config is null").getHadoopImagesVersion(); + this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/singlenode-spark-iceberg-rest"); } @Override @@ -75,15 +91,37 @@ public void extendEnvironment(Environment.Builder builder) builder.configureContainer(TESTS, dockerContainer -> dockerContainer // Binding instead of copying for avoiding OutOfMemoryError https://github.com/testcontainers/testcontainers-java/issues/2863 .withFileSystemBind(HIVE_JDBC_PROVIDER.getParent(), "/docker/jdbc", BindMode.READ_ONLY)); + + // Initialize buckets in Minio + FileAttribute> posixFilePermissions = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")); + Path minioBucketDirectory; + try { + minioBucketDirectory = Files.createTempDirectory("test-bucket-contents", posixFilePermissions); + minioBucketDirectory.toFile().deleteOnExit(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + builder.configureContainer(MINIO_CONTAINER_NAME, container -> + container.withCopyFileToContainer(forHostPath(minioBucketDirectory), "/data/" + S3_BUCKET_NAME)); + + configureTempto(builder, configDir); } @SuppressWarnings("resource") private DockerContainer createRESTContainer() { DockerContainer container = new DockerContainer(REST_SERVER_IMAGE, REST_CONTAINER_NAME) + .withEnv("CATALOG_INCLUDE__CREDENTIALS", "true") .withEnv("CATALOG_WAREHOUSE", CATALOG_WAREHOUSE) + .withEnv("CATALOG_URI", "jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory") + .withEnv("CATALOG_IO__IMPL", "org.apache.iceberg.aws.s3.S3FileIO") + .withEnv("AWS_REGION", "us-east-1") + .withEnv("CATALOG_S3_ACCESS__KEY__ID", "minio-access-key") + .withEnv("CATALOG_S3_SECRET__ACCESS__KEY", "minio-secret-key") + .withEnv("CATALOG_S3_ENDPOINT", "http://minio:9080") + .withEnv("CATALOG_S3_PATH__STYLE__ACCESS", "true") .withEnv("REST_PORT", Integer.toString(REST_SERVER_PORT)) - .withEnv("HADOOP_USER_NAME", "hive") .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) .waitingFor(forSelectedPorts(REST_SERVER_PORT)); @@ -94,8 +132,11 @@ private DockerContainer createRESTContainer() @SuppressWarnings("resource") private DockerContainer createSparkContainer() { + System.out.println(hadoopImagesVersion); DockerContainer container = new DockerContainer("ghcr.io/trinodb/testing/spark3-iceberg:" + hadoopImagesVersion, SPARK_CONTAINER_NAME) - .withEnv("HADOOP_USER_NAME", "hive") + .withEnv("AWS_REGION", "us-east-1") + .withEnv("AWS_ACCESS_KEY_ID", "minio-access-key") + .withEnv("AWS_SECRET_ACCESS_KEY", "minio-secret-key") .withCopyFileToContainer( forHostPath(dockerFiles.getDockerFilesHostPath( "conf/environment/singlenode-spark-iceberg-rest/spark-defaults.conf")), diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-jdbc-catalog/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-jdbc-catalog/tempto-configuration.yaml new file mode 100644 index 000000000000..728f36d8c831 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-jdbc-catalog/tempto-configuration.yaml @@ -0,0 +1,2 @@ +iceberg: + default_schema_location: hdfs://hadoop-master:9000/user/hive/warehouse/default diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-nessie/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-nessie/tempto-configuration.yaml new file mode 100644 index 000000000000..728f36d8c831 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-nessie/tempto-configuration.yaml @@ -0,0 +1,2 @@ +iceberg: + default_schema_location: hdfs://hadoop-master:9000/user/hive/warehouse/default diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/iceberg.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/iceberg.properties index 9f88080be70e..d1360444c170 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/iceberg.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/iceberg.properties @@ -2,4 +2,10 @@ connector.name=iceberg iceberg.catalog.type=rest iceberg.rest-catalog.uri=http://iceberg-with-rest:8181/ iceberg.allowed-extra-properties=custom.table-property -fs.hadoop.enabled=true +fs.native-s3.enabled=true +fs.hadoop.enabled=false +s3.region=us-east-1 +s3.aws-access-key=minio-access-key +s3.aws-secret-key=minio-secret-key +s3.endpoint=http://minio:9080/ +s3.path-style-access=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/spark-defaults.conf b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/spark-defaults.conf index be92ee99931e..d276e8116ffc 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/spark-defaults.conf +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/spark-defaults.conf @@ -3,7 +3,7 @@ spark.sql.defaultCatalog=iceberg_test spark.sql.catalog.iceberg_test=org.apache.iceberg.spark.SparkCatalog spark.sql.catalog.iceberg_test.catalog-impl=org.apache.iceberg.rest.RESTCatalog spark.sql.catalog.iceberg_test.uri=http://iceberg-with-rest:8181/ +spark.sql.catalog.iceberg_test.s3.endpoint=http://minio:9080 +spark.sql.catalog.iceberg_test.s3.path-style-access=true ; disabling caching allows us to run spark queries interchangeably with trino's spark.sql.catalog.iceberg_test.cache-enabled=false -spark.sql.catalog.iceberg_test.warehouse=hdfs://hadoop-master:9000/user/hive/warehouse -spark.hadoop.fs.defaultFS=hdfs://hadoop-master:9000 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/tempto-configuration.yaml new file mode 100644 index 000000000000..febf0cb5e0e0 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg-rest/tempto-configuration.yaml @@ -0,0 +1,2 @@ +iceberg: + default_schema_location: s3://test-bucket/default diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg/tempto-configuration.yaml new file mode 100644 index 000000000000..728f36d8c831 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-spark-iceberg/tempto-configuration.yaml @@ -0,0 +1,2 @@ +iceberg: + default_schema_location: hdfs://hadoop-master:9000/user/hive/warehouse/default diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java index df814c6cb20c..7d749ec896f7 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergSparkCompatibility.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; import com.google.inject.Inject; +import com.google.inject.name.Named; import io.airlift.concurrent.MoreFutures; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreClient; import io.trino.tempto.AfterMethodWithContext; @@ -87,6 +88,9 @@ public class TestIcebergSparkCompatibility extends ProductTest { + @Inject + @Named("iceberg.default_schema_location") + private String schemaLocation; @Inject private HdfsClient hdfsClient; @Inject @@ -99,7 +103,7 @@ public void setup() { metastoreClient = testHiveMetastoreClientFactory.createMetastoreClient(); // Create 'default' schema if it doesn't exist because JDBC catalog doesn't have such schema - onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS iceberg.default WITH (location = 'hdfs://hadoop-master:9000/user/hive/warehouse/default')"); + onTrino().executeQuery("CREATE SCHEMA IF NOT EXISTS iceberg.default WITH (location = '" + schemaLocation + "')"); } @AfterMethodWithContext diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java index 0516d4caffb8..abf78db4fccd 100644 --- a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java @@ -32,13 +32,15 @@ public IcebergRestCatalogBackendContainer( String minioSessionToken) { super( - "tabulario/iceberg-rest:1.5.0", + // TODO Pin version once Iceberg community releases it + "apache/iceberg-rest-fixture:latest", "iceberg-rest", ImmutableSet.of(8181), ImmutableMap.of(), ImmutableMap.of( "CATALOG_INCLUDE__CREDENTIALS", "true", "CATALOG_WAREHOUSE", warehouseLocation, + "CATALOG_URI", "jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory", "CATALOG_IO__IMPL", "org.apache.iceberg.aws.s3.S3FileIO", "AWS_REGION", MINIO_REGION, "CATALOG_S3_ACCESS__KEY__ID", minioAccessKey,