diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96f1bfb5bf95..dc6013fc1e5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -525,6 +525,8 @@ jobs: build-pt: runs-on: ubuntu-latest + outputs: + have_azure_secrets: ${{ steps.check-secrets.outputs.have_azure_secrets }} steps: - uses: actions/checkout@v2 with: @@ -536,6 +538,20 @@ jobs: distribution: 'zulu' java-version: 11 cache: 'maven' + - name: Check secrets + run: | + if [[ "${{ secrets.AZURE_ABFS_CONTAINER }}" != "" && \ + "${{ secrets.AZURE_ABFS_ACCOUNT }}" != "" && \ + "${{ secrets.AZURE_ABFS_ACCESSKEY }}" != "" + ]]; \ + then + echo "Secrets to run product tests were configured in the repo" + echo "::set-output name=have_azure_secrets::true" + else + echo "Secrets to run product tests were not configured in the repo" + echo "::set-output name=have_azure_secrets::false" + fi + id: check-secrets - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" @@ -718,3 +734,63 @@ jobs: name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) path: testing/trino-product-tests/target/reports/**/testng-results.xml retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + + azure-pt: + runs-on: ubuntu-latest + needs: build-pt + if: needs.build-pt.outputs.have_azure_secrets == 'true' + strategy: + fail-fast: false + matrix: + config: + - hdp3 + suite: + - suite-azure + jdk: + - 11 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent + - uses: actions/setup-java@v2 + with: + distribution: 'zulu' + java-version: 11 + - name: Product tests artifact + uses: actions/download-artifact@v2 + with: + name: product tests and server tarball + - name: Fix artifact permissions + run: | + find . -type f -name \*-executable.jar -exec chmod 0777 {} \; + - name: Product Tests + env: + ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }} + ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }} + ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }} + run: | + testing/bin/ptl suite run \ + --suite ${{ matrix.suite }} \ + --config config-${{ matrix.config }} \ + --bind=off --logs-dir logs/ --timeout 2h \ + --trino-jdk-version zulu_${{ matrix.jdk }} + - name: Upload test logs and results + uses: actions/upload-artifact@v2 + # Upload all test reports only on failure, because the artifacts are large + if: failure() + with: + name: result pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) + path: | + testing/trino-product-tests/target/* + logs/* + - name: Upload test report + uses: actions/upload-artifact@v2 + # Always upload the test report for the annotate.yml workflow, + # but only the single XML file to keep the artifact small + if: always() + with: + # Name prefix is checked in the `Annotate checks` workflow + name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) + path: testing/trino-product-tests/target/reports/**/testng-results.xml + retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java new file mode 100644 index 000000000000..6479552fbd0d --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAzure.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.Hadoop; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import javax.inject.Inject; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; + +import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.WORKER; +import static io.trino.tests.product.launcher.env.common.Hadoop.CONTAINER_HADOOP_INIT_D; +import static java.nio.file.attribute.PosixFilePermissions.fromString; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class EnvMultinodeAzure + extends EnvironmentProvider +{ + private final DockerFiles dockerFiles; + private final DockerFiles.ResourceProvider configDir; + private final String hadoopBaseImage; + private final String hadoopImagesVersion; + + @Inject + public EnvMultinodeAzure(DockerFiles dockerFiles, StandardMultinode standardMultinode, Hadoop hadoop, EnvironmentConfig environmentConfig) + { + super(ImmutableList.of(standardMultinode, hadoop)); + this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); + configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/multinode-azure"); + requireNonNull(environmentConfig, "environmentConfig is null"); + hadoopBaseImage = environmentConfig.getHadoopBaseImage(); + hadoopImagesVersion = environmentConfig.getHadoopImagesVersion(); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + String dockerImageName = hadoopBaseImage + ":" + hadoopImagesVersion; + + builder.configureContainer(HADOOP, container -> { + container.setDockerImageName(dockerImageName); + container.withCopyFileToContainer( + forHostPath(getCoreSiteOverrideXml()), + "/docker/presto-product-tests/conf/environment/multinode-azure/core-site-overrides.xml"); + container.withCopyFileToContainer( + forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-azure/apply-azure-config.sh")), + CONTAINER_HADOOP_INIT_D + "apply-azure-config.sh"); + }); + + builder.configureContainer(COORDINATOR, container -> container + .withEnv("ABFS_ACCOUNT", requireEnv("ABFS_ACCOUNT")) + .withEnv("ABFS_ACCESS_KEY", requireEnv("ABFS_ACCESS_KEY"))); + + builder.configureContainer(WORKER, container -> container + .withEnv("ABFS_ACCOUNT", requireEnv("ABFS_ACCOUNT")) + .withEnv("ABFS_ACCESS_KEY", requireEnv("ABFS_ACCESS_KEY"))); + + builder.configureContainer(TESTS, container -> container + .withEnv("ABFS_CONTAINER", requireEnv("ABFS_CONTAINER")) + .withEnv("ABFS_ACCOUNT", requireEnv("ABFS_ACCOUNT"))); + + builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); + } + + private Path getCoreSiteOverrideXml() + { + try { + String coreSite = Files.readString(configDir.getPath("core-site-overrides-template.xml")) + .replace("%ABFS_ACCOUNT%", requireEnv("ABFS_ACCOUNT")) + .replace("%ABFS_ACCESS_KEY%", requireEnv("ABFS_ACCESS_KEY")); + File coreSiteXml = Files.createTempFile("core-site", ".xml", PosixFilePermissions.asFileAttribute(fromString("rwxrwxrwx"))).toFile(); + coreSiteXml.deleteOnExit(); + Files.writeString(coreSiteXml.toPath(), coreSite); + return coreSiteXml.toPath(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String requireEnv(String variable) + { + return requireNonNull(System.getenv(variable), () -> "environment variable not set: " + variable); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite1.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite1.java index ed0925056682..b67a26dec62a 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite1.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite1.java @@ -31,7 +31,7 @@ public List getTestRuns(EnvironmentConfig config) { return ImmutableList.of( testOnEnvironment(EnvMultinode.class) - .withExcludedGroups("large_query", "storage_formats", "storage_formats_detailed", "profile_specific_tests", "tpcds", "hive_compression") + .withExcludedGroups("large_query", "storage_formats", "storage_formats_detailed", "profile_specific_tests", "tpcds", "hive_compression", "azure") .build()); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite3.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite3.java index 33abb9abd04d..eeec64f77a6e 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite3.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite3.java @@ -36,6 +36,7 @@ public List getTestRuns(EnvironmentConfig config) return ImmutableList.of( testOnEnvironment(EnvMultinodeTls.class) .withGroups("configured_features", "smoke", "cli", "group-by", "join", "tls") + .withExcludedGroups("azure") .build(), testOnEnvironment(EnvMultinodeTlsKerberos.class) .withGroups("configured_features", "cli", "group-by", "join", "tls") diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteAzure.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteAzure.java new file mode 100644 index 000000000000..a995c99e0ab2 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteAzure.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.suite.suites; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.env.environment.EnvMultinodeAzure; +import io.trino.tests.product.launcher.suite.Suite; +import io.trino.tests.product.launcher.suite.SuiteTestRun; + +import java.util.List; + +import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; + +public class SuiteAzure + extends Suite +{ + @Override + public List getTestRuns(EnvironmentConfig config) + { + return ImmutableList.of( + testOnEnvironment(EnvMultinodeAzure.class) + .withGroups("configured_features", "azure") + .build()); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/apply-azure-config.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/apply-azure-config.sh new file mode 100755 index 000000000000..a4a30ddc1cf1 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/apply-azure-config.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -exuo pipefail + +echo "Applying HDP3 core-site configuration overrides" +apply-site-xml-override /etc/hadoop/conf/core-site.xml "/docker/presto-product-tests/conf/environment/multinode-azure/core-site-overrides.xml" diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/core-site-overrides-template.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/core-site-overrides-template.xml new file mode 100644 index 000000000000..4419587c5b1f --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/core-site-overrides-template.xml @@ -0,0 +1,8 @@ + + + + fs.azure.account.key.%ABFS_ACCOUNT%.dfs.core.windows.net + %ABFS_ACCESS_KEY% + + + diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties new file mode 100644 index 000000000000..8a9f623b07ec --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties @@ -0,0 +1,13 @@ +connector.name=hive +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.azure.abfs-storage-account=${ENV:ABFS_ACCOUNT} +hive.azure.abfs-access-key=${ENV:ABFS_ACCESS_KEY} +hive.non-managed-table-writes-enabled=true +hive.allow-add-column=true +hive.allow-drop-column=true +hive.allow-rename-column=true +hive.allow-comment-table=true +hive.allow-drop-table=true +hive.allow-rename-table=true +hive.translate-hive-views=true diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java index 389336d1df5d..a50383afe72f 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestGroups.java @@ -51,6 +51,7 @@ public final class TestGroups public static final String HIVE_ICEBERG_REDIRECTIONS = "hive_iceberg_redirections"; public static final String AUTHORIZATION = "authorization"; public static final String HIVE_COERCION = "hive_coercion"; + public static final String AZURE = "azure"; public static final String CASSANDRA = "cassandra"; public static final String SQL_SERVER = "sqlserver"; public static final String LDAP = "ldap"; diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestSyncPartitionMetadata.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestSyncPartitionMetadata.java similarity index 68% rename from testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestSyncPartitionMetadata.java rename to testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestSyncPartitionMetadata.java index 1faca8af75df..20c01248bd13 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestSyncPartitionMetadata.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestSyncPartitionMetadata.java @@ -13,60 +13,53 @@ */ package io.trino.tests.product.hive; -import com.google.inject.Inject; -import com.google.inject.name.Named; +import io.trino.tempto.AfterTestWithContext; +import io.trino.tempto.BeforeTestWithContext; import io.trino.tempto.ProductTest; import io.trino.tempto.assertions.QueryAssert; -import io.trino.tempto.fulfillment.table.hive.HiveDataSource; -import io.trino.tempto.hadoop.hdfs.HdfsClient; -import io.trino.tempto.internal.hadoop.hdfs.HdfsDataSourceWriter; import io.trino.tempto.query.QueryResult; -import io.trino.testng.services.Flaky; -import org.testng.annotations.Test; +import static com.google.common.collect.Iterables.getOnlyElement; import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.tempto.assertions.QueryAssert.assertThat; -import static io.trino.tempto.fulfillment.table.hive.InlineDataSource.createResourceDataSource; -import static io.trino.tests.product.TestGroups.HIVE_PARTITIONING; -import static io.trino.tests.product.TestGroups.SMOKE; -import static io.trino.tests.product.TestGroups.TRINO_JDBC; -import static io.trino.tests.product.hive.HiveProductTest.ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE; -import static io.trino.tests.product.hive.HiveProductTest.ERROR_COMMITTING_WRITE_TO_HIVE_MATCH; import static io.trino.tests.product.hive.util.TableLocationUtils.getTableLocation; +import static io.trino.tests.product.utils.QueryExecutors.onHive; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThatThrownBy; -public class TestSyncPartitionMetadata +/** + * Please add @Test with groups in the subclass + */ +public abstract class BaseTestSyncPartitionMetadata extends ProductTest { - @Inject - @Named("databases.hive.warehouse_directory_path") - private String warehouseDirectory; - - @Inject - private HdfsClient hdfsClient; + @BeforeTestWithContext + public void setUp() + { + onHdfs("-rm -f -r " + schemaLocation()); + onHdfs("-mkdir -p " + schemaLocation()); + } - @Inject - private HdfsDataSourceWriter hdfsDataSourceWriter; + @AfterTestWithContext + public void tearDown() + { + onHdfs("-rm -f -r " + schemaLocation()); + } - @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testAddPartition() { String tableName = "test_sync_partition_metadata_add_partition"; - prepare(hdfsClient, hdfsDataSourceWriter, tableName); + prepare(tableName); onTrino().executeQuery("CALL system.sync_partition_metadata('default', '" + tableName + "', 'ADD')"); assertPartitions(tableName, row("a", "1"), row("b", "2"), row("f", "9")); assertQueryFailure(() -> onTrino().executeQuery("SELECT payload, col_x, col_y FROM " + tableName + " ORDER BY 1, 2, 3 ASC")) - .hasMessageContaining(format("Partition location does not exist: hdfs://hadoop-master:9000%s/%s/col_x=b/col_y=2", warehouseDirectory, tableName)); + .hasMessageMatching(format(".*Partition location does not exist: .*%s/col_x=b/col_y=2", tableLocation(tableName))); cleanup(tableName); } - @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testAddPartitionContainingCharactersThatNeedUrlEncoding() { String tableName = "test_sync_partition_metadata_add_partition_urlencode"; @@ -104,12 +97,10 @@ public void testAddPartitionContainingCharactersThatNeedUrlEncoding() cleanup(tableName); } - @Test(groups = {HIVE_PARTITIONING, SMOKE}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testDropPartition() { String tableName = "test_sync_partition_metadata_drop_partition"; - prepare(hdfsClient, hdfsDataSourceWriter, tableName); + prepare(tableName); onTrino().executeQuery("CALL system.sync_partition_metadata('default', '" + tableName + "', 'DROP')"); assertPartitions(tableName, row("a", "1")); @@ -118,8 +109,6 @@ public void testDropPartition() cleanup(tableName); } - @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testDropPartitionContainingCharactersThatNeedUrlEncoding() { String tableName = "test_sync_partition_metadata_drop_partition_urlencode"; @@ -160,12 +149,10 @@ public void testDropPartitionContainingCharactersThatNeedUrlEncoding() cleanup(tableName); } - @Test(groups = {HIVE_PARTITIONING, SMOKE}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testFullSyncPartition() { String tableName = "test_sync_partition_metadata_add_drop_partition"; - prepare(hdfsClient, hdfsDataSourceWriter, tableName); + prepare(tableName); onTrino().executeQuery("CALL system.sync_partition_metadata('default', '" + tableName + "', 'FULL')"); assertPartitions(tableName, row("a", "1"), row("f", "9")); @@ -174,12 +161,10 @@ public void testFullSyncPartition() cleanup(tableName); } - @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testInvalidSyncMode() { String tableName = "test_repair_invalid_mode"; - prepare(hdfsClient, hdfsDataSourceWriter, tableName); + prepare(tableName); assertQueryFailure(() -> onTrino().executeQuery("CALL system.sync_partition_metadata('default', '" + tableName + "', 'INVALID')")) .hasMessageMatching("Query failed (.*): Invalid partition metadata sync mode: INVALID"); @@ -187,32 +172,33 @@ public void testInvalidSyncMode() cleanup(tableName); } - @Test(groups = {HIVE_PARTITIONING, SMOKE}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testMixedCasePartitionNames() { String tableName = "test_sync_partition_mixed_case"; - prepare(hdfsClient, hdfsDataSourceWriter, tableName); + prepare(tableName); String tableLocation = tableLocation(tableName); - HiveDataSource dataSource = createResourceDataSource(tableName, "io/trino/tests/product/hive/data/single_int_column/data.orc"); - hdfsDataSourceWriter.ensureDataOnHdfs(tableLocation + "/col_x=h/col_Y=11", dataSource); - hdfsClient.createDirectory(tableLocation + "/COL_X=UPPER/COL_Y=12"); - hdfsDataSourceWriter.ensureDataOnHdfs(tableLocation + "/COL_X=UPPER/COL_Y=12", dataSource); + + String dataSource = generateOrcFile(); + onHdfs(format("-mkdir -p %s/col_x=h/col_Y=11", tableLocation)); + onHdfs(format("-cp %s %s/col_x=h/col_Y=11", dataSource, tableLocation)); + + onHdfs(format("-mkdir -p %s/COL_X=UPPER/COL_Y=12", tableLocation)); + onHdfs(format("-cp %s %s/COL_X=UPPER/COL_Y=12", dataSource, tableLocation)); onTrino().executeQuery("CALL system.sync_partition_metadata('default', '" + tableName + "', 'FULL', false)"); assertPartitions(tableName, row("UPPER", "12"), row("a", "1"), row("f", "9"), row("g", "10"), row("h", "11")); assertData(tableName, row(1, "a", "1"), row(42, "UPPER", "12"), row(42, "f", "9"), row(42, "g", "10"), row(42, "h", "11")); } - @Test(groups = {HIVE_PARTITIONING, SMOKE}) - @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) public void testConflictingMixedCasePartitionNames() { String tableName = "test_sync_partition_mixed_case"; - prepare(hdfsClient, hdfsDataSourceWriter, tableName); - HiveDataSource dataSource = createResourceDataSource(tableName, "io/trino/tests/product/hive/data/single_int_column/data.orc"); + String tableLocation = tableLocation(tableName); + prepare(tableName); + String dataSource = generateOrcFile(); // this conflicts with a partition that already exits in the metastore - hdfsDataSourceWriter.ensureDataOnHdfs(tableLocation(tableName) + "/COL_X=a/cOl_y=1", dataSource); + onHdfs(format("-mkdir -p %s/COL_X=a/cOl_y=1", tableLocation)); + onHdfs(format("-cp %s %s/COL_X=a/cOl_y=1", dataSource, tableLocation)); assertThatThrownBy(() -> onTrino().executeQuery("CALL system.sync_partition_metadata('default', '" + tableName + "', 'ADD', false)")) .hasMessageContaining(format("One or more partitions already exist for table 'default.%s'", tableName)); @@ -221,36 +207,56 @@ public void testConflictingMixedCasePartitionNames() private String tableLocation(String tableName) { - return warehouseDirectory + '/' + tableName; + return schemaLocation() + '/' + tableName; } - private void prepare(HdfsClient hdfsClient, HdfsDataSourceWriter hdfsDataSourceWriter, String tableName) + protected abstract String schemaLocation(); + + private void prepare(String tableName) { + String tableLocation = tableLocation(tableName); onTrino().executeQuery("DROP TABLE IF EXISTS " + tableName); + onHdfs("-rm -f -r " + tableLocation); + onHdfs("-mkdir -p " + tableLocation); - onTrino().executeQuery("CREATE TABLE " + tableName + " (payload bigint, col_x varchar, col_y varchar) WITH (format = 'ORC', partitioned_by = ARRAY[ 'col_x', 'col_y' ])"); + onHive().executeQuery("CREATE TABLE " + tableName + " (payload bigint) PARTITIONED BY (col_x string, col_y string) STORED AS ORC LOCATION '" + tableLocation + "'"); onTrino().executeQuery("INSERT INTO " + tableName + " VALUES (1, 'a', '1'), (2, 'b', '2')"); - String tableLocation = tableLocation(tableName); + String filePath = generateOrcFile(); + // remove partition col_x=b/col_y=2 - hdfsClient.delete(tableLocation + "/col_x=b/col_y=2"); + onHdfs(format("-rm -f -r %s/col_x=b/col_y=2", tableLocation)); // add partition directory col_x=f/col_y=9 with single_int_column/data.orc file - hdfsClient.createDirectory(tableLocation + "/col_x=f/col_y=9"); - HiveDataSource dataSource = createResourceDataSource(tableName, "io/trino/tests/product/hive/data/single_int_column/data.orc"); - hdfsDataSourceWriter.ensureDataOnHdfs(tableLocation + "/col_x=f/col_y=9", dataSource); + onHdfs(format("-mkdir -p %s/col_x=f/col_y=9", tableLocation)); + onHdfs(format("-cp %s %s/col_x=f/col_y=9", filePath, tableLocation)); + // should only be picked up when not in case sensitive mode - hdfsClient.createDirectory(tableLocation + "/COL_X=g/col_y=10"); - hdfsDataSourceWriter.ensureDataOnHdfs(tableLocation + "/COL_X=g/col_y=10", dataSource); + onHdfs(format("-mkdir -p %s/COL_X=g/col_y=10", tableLocation)); + onHdfs(format("-cp %s %s/COL_X=g/col_y=10", filePath, tableLocation)); // add invalid partition path - hdfsClient.createDirectory(tableLocation + "/col_x=d"); - hdfsClient.createDirectory(tableLocation + "/col_y=3/col_x=h"); - hdfsClient.createDirectory(tableLocation + "/col_y=3"); - hdfsClient.createDirectory(tableLocation + "/xyz"); + onHdfs(format("-mkdir -p %s/col_x=d", tableLocation)); + onHdfs(format("-mkdir -p %s/col_y=3/col_x=h", tableLocation)); + onHdfs(format("-mkdir -p %s/col_y=3", tableLocation)); + onHdfs(format("-mkdir -p %s/xyz", tableLocation)); assertPartitions(tableName, row("a", "1"), row("b", "2")); } + // Drop and create a table. Then, return single ORC file path + private String generateOrcFile() + { + onTrino().executeQuery("DROP TABLE IF EXISTS single_int_column"); + onTrino().executeQuery("CREATE TABLE single_int_column (payload bigint) WITH (format = 'ORC')"); + onTrino().executeQuery("INSERT INTO single_int_column VALUES (42)"); + return getOnlyElement(onTrino().executeQuery("SELECT \"$path\" FROM single_int_column").row(0)).toString(); + } + + private void onHdfs(String command) + { + onHive().executeQuery("dfs " + command); + } + private static void cleanup(String tableName) { onTrino().executeQuery("DROP TABLE " + tableName); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java new file mode 100644 index 000000000000..3714e4582a55 --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAbfsSyncPartitionMetadata.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.hive; + +import io.trino.testng.services.Flaky; +import org.testng.annotations.Test; + +import static io.trino.tests.product.TestGroups.AZURE; +import static io.trino.tests.product.hive.HiveProductTest.ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE; +import static io.trino.tests.product.hive.HiveProductTest.ERROR_COMMITTING_WRITE_TO_HIVE_MATCH; +import static io.trino.tests.product.hive.util.TemporaryHiveTable.randomTableSuffix; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class TestAbfsSyncPartitionMetadata + extends BaseTestSyncPartitionMetadata +{ + private final String schema = "test_" + randomTableSuffix(); + + @Override + protected String schemaLocation() + { + String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); + String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); + return format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testAddPartition() + { + super.testAddPartition(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testAddPartitionContainingCharactersThatNeedUrlEncoding() + { + super.testAddPartitionContainingCharactersThatNeedUrlEncoding(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testDropPartition() + { + super.testDropPartition(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testDropPartitionContainingCharactersThatNeedUrlEncoding() + { + super.testDropPartitionContainingCharactersThatNeedUrlEncoding(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testFullSyncPartition() + { + super.testFullSyncPartition(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testInvalidSyncMode() + { + super.testInvalidSyncMode(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testMixedCasePartitionNames() + { + super.testMixedCasePartitionNames(); + } + + @Test(groups = AZURE) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testConflictingMixedCasePartitionNames() + { + super.testConflictingMixedCasePartitionNames(); + } +} diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHdfsSyncPartitionMetadata.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHdfsSyncPartitionMetadata.java new file mode 100644 index 000000000000..4cf92df6bb5f --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHdfsSyncPartitionMetadata.java @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.hive; + +import com.google.inject.Inject; +import com.google.inject.name.Named; +import io.trino.testng.services.Flaky; +import org.testng.annotations.Test; + +import static io.trino.tests.product.TestGroups.HIVE_PARTITIONING; +import static io.trino.tests.product.TestGroups.SMOKE; +import static io.trino.tests.product.TestGroups.TRINO_JDBC; +import static io.trino.tests.product.hive.HiveProductTest.ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE; +import static io.trino.tests.product.hive.HiveProductTest.ERROR_COMMITTING_WRITE_TO_HIVE_MATCH; +import static io.trino.tests.product.hive.util.TemporaryHiveTable.randomTableSuffix; +import static java.lang.String.format; + +public class TestHdfsSyncPartitionMetadata + extends BaseTestSyncPartitionMetadata +{ + @Inject + @Named("databases.hive.warehouse_directory_path") + private String warehouseDirectory; + + private final String schema = "test_" + randomTableSuffix(); + + @Override + protected String schemaLocation() + { + return format("%s/%s", warehouseDirectory, schema); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testAddPartition() + { + super.testAddPartition(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testAddPartitionContainingCharactersThatNeedUrlEncoding() + { + super.testAddPartitionContainingCharactersThatNeedUrlEncoding(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testDropPartition() + { + super.testDropPartition(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testDropPartitionContainingCharactersThatNeedUrlEncoding() + { + super.testDropPartitionContainingCharactersThatNeedUrlEncoding(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testFullSyncPartition() + { + super.testFullSyncPartition(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE, TRINO_JDBC}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testInvalidSyncMode() + { + super.testInvalidSyncMode(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testMixedCasePartitionNames() + { + super.testMixedCasePartitionNames(); + } + + @Test(groups = {HIVE_PARTITIONING, SMOKE}) + @Flaky(issue = ERROR_COMMITTING_WRITE_TO_HIVE_ISSUE, match = ERROR_COMMITTING_WRITE_TO_HIVE_MATCH) + @Override + public void testConflictingMixedCasePartitionNames() + { + super.testConflictingMixedCasePartitionNames(); + } +}