From adf8331a815d867044d89eb89aa5da5352b32713 Mon Sep 17 00:00:00 2001 From: Pramod Satya Date: Wed, 26 Mar 2025 15:25:56 -0700 Subject: [PATCH] Run presto-native-tests with DWRF and PARQUET --- .../prestocpp-linux-build-and-unit-test.yml | 5 +++++ presto-native-tests/README.md | 7 +++++-- .../TestDistributedEngineOnlyQueries.java | 15 +++++++++++---- .../TestOrderByQueries.java | 15 +++++++++++---- .../TestRepartitionQueries.java | 15 +++++++++++---- .../TestRepartitionQueriesWithSmallPages.java | 16 ++++++++++++---- .../TestWindowQueries.java | 17 ++++++++++++----- 7 files changed, 67 insertions(+), 23 deletions(-) diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml index 262e89d2d6e9e..219b4db826c85 100644 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -168,6 +168,10 @@ jobs: prestocpp-linux-presto-native-tests: needs: prestocpp-linux-build-for-test runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + storage-format: [ "PARQUET", "DWRF" ] container: image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84 env: @@ -230,6 +234,7 @@ jobs: mvn test \ ${MAVEN_TEST} \ -pl 'presto-native-tests' \ + -DstorageFormat=${{ matrix.storage-format }} \ -Dtest="${TESTCLASSES}" \ -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ -DDATA_DIR=${RUNNER_TEMP} \ diff --git a/presto-native-tests/README.md b/presto-native-tests/README.md index a882b2a33b463..926d011a07979 100644 --- a/presto-native-tests/README.md +++ b/presto-native-tests/README.md @@ -8,13 +8,16 @@ The following command can be used to run all tests in this module: ``` mvn test -pl 'presto-native-tests' + -DstorageFormat="PARQUET" -Dtest="com.facebook.presto.nativetests.Test*" -Duser.timezone=America/Bahia_Banderas -DPRESTO_SERVER=${PRESTO_HOME}/presto-native-execution/cmake-build-debug/presto_cpp/main/presto_server -DWORKER_COUNT=${WORKER_COUNT} -T1C ``` -Please update JVM argument `PRESTO_SERVER` to point to the Presto C++ worker -binary `presto_server`. +Update the following JVM arguments: +1. `PRESTO_SERVER`: Points to the Presto C++ worker binary, `presto_server`. +2. `storageFormat`: File format of TPC-H tables used in tests, `DWRF` and +`PARQUET` formats are supported. ## Adding new tests diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java index e83e4a2a6e589..e0ae26b31fd65 100644 --- a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestDistributedEngineOnlyQueries.java @@ -19,6 +19,7 @@ import com.facebook.presto.tests.AbstractTestEngineOnlyQueries; import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; import java.time.LocalDate; @@ -34,20 +35,26 @@ public class TestDistributedEngineOnlyQueries { private static final String timeTypeUnsupportedError = ".*Failed to parse type \\[time.*"; - private static final String storageFormat = "PARQUET"; - + @Parameters("storageFormat") @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), storageFormat); + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); } + @Parameters("storageFormat") @Override protected void createTables() { try { + String storageFormat = System.getProperty("storageFormat"); QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); - NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + if (storageFormat.equals("DWRF")) { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); + } + else { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + } javaQueryRunner.close(); } catch (Exception e) { diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java index 1f8f378e2bdcf..ec8f32b2261fd 100644 --- a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestOrderByQueries.java @@ -18,25 +18,32 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestOrderByQueries; import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; public class TestOrderByQueries extends AbstractTestOrderByQueries { - private static final String storageFormat = "PARQUET"; - + @Parameters("storageFormat") @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), storageFormat); + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); } + @Parameters("storageFormat") @Override protected void createTables() { try { + String storageFormat = System.getProperty("storageFormat"); QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); - NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + if (storageFormat.equals("DWRF")) { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); + } + else { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + } javaQueryRunner.close(); } catch (Exception e) { diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java index 2bb1125f63ca5..ebc8f19105c09 100644 --- a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueries.java @@ -18,24 +18,31 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestRepartitionQueries; import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Parameters; public class TestRepartitionQueries extends AbstractTestRepartitionQueries { - private static final String storageFormat = "PARQUET"; - + @Parameters("storageFormat") @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), storageFormat); + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); } + @Parameters("storageFormat") @Override protected void createTables() { try { + String storageFormat = System.getProperty("storageFormat"); QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); - NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + if (storageFormat.equals("DWRF")) { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); + } + else { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + } javaQueryRunner.close(); } catch (Exception e) { diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java index ba48b071c58b2..5e354f3cd3cf6 100644 --- a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestRepartitionQueriesWithSmallPages.java @@ -18,26 +18,34 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestRepartitionQueries; import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Parameters; public class TestRepartitionQueriesWithSmallPages extends AbstractTestRepartitionQueries { - private static final String storageFormat = "PARQUET"; - + @Parameters("storageFormat") @Override protected QueryRunner createQueryRunner() throws Exception { return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( // Use small SerializedPages to force flushing - ImmutableMap.of("driver.max-page-partitioning-buffer-size", "200B"), storageFormat); + ImmutableMap.of("driver.max-page-partitioning-buffer-size", "200B"), + System.getProperty("storageFormat")); } + @Parameters("storageFormat") @Override protected void createTables() { try { + String storageFormat = System.getProperty("storageFormat"); QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); - NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + if (storageFormat.equals("DWRF")) { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); + } + else { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + } javaQueryRunner.close(); } catch (Exception e) { diff --git a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java index 634c3b5de1e89..d51e50a1b8361 100644 --- a/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java +++ b/presto-native-tests/src/test/java/com.facebook.presto.nativetests/TestWindowQueries.java @@ -18,6 +18,7 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestWindowQueries; import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; public class TestWindowQueries @@ -25,20 +26,26 @@ public class TestWindowQueries { private static final String frameTypeDiffersError = ".*Window frame of type RANGE does not match types of the ORDER BY and frame column.*"; - private String storageFormat = "PARQUET"; - + @Parameters("storageFormat") @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), storageFormat); + return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); } + @Parameters("storageFormat") @Override protected void createTables() { try { - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); - NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + String storageFormat = System.getProperty("storageFormat"); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + if (storageFormat.equals("DWRF")) { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); + } + else { + NativeQueryRunnerUtils.createAllTables(javaQueryRunner, false); + } javaQueryRunner.close(); } catch (Exception e) {