diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/HiveExternalWorkerQueryRunner.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/HiveExternalWorkerQueryRunner.java index f9b42737882e3..2e82f37681013 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/HiveExternalWorkerQueryRunner.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/HiveExternalWorkerQueryRunner.java @@ -29,12 +29,12 @@ public static void main(String[] args) Logging.initialize(); // Create tables before launching distributed runner. - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(false); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder().build(); NativeQueryRunnerUtils.createAllTables(javaQueryRunner); javaQueryRunner.close(); // Launch distributed runner. - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(false, false, false, false); + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder().build(); Thread.sleep(10); Logger log = Logger.get(DistributedQueryRunner.class); log.info("======== SERVER STARTED ========"); diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/IcebergExternalWorkerQueryRunner.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/IcebergExternalWorkerQueryRunner.java index ff7d871f3a432..143cef6e3fcde 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/IcebergExternalWorkerQueryRunner.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/IcebergExternalWorkerQueryRunner.java @@ -29,12 +29,14 @@ public static void main(String[] args) Logging.initialize(); // Create tables before launching distributed runner. - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner(false); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder().build(); NativeQueryRunnerUtils.createAllIcebergTables(javaQueryRunner); javaQueryRunner.close(); // Launch distributed runner. - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(true, false); + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setUseThrift(true) + .build(); Thread.sleep(10); Logger log = Logger.get(DistributedQueryRunner.class); log.info("======== SERVER STARTED ========"); diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java index 8a0c0facf96d5..f9db77b539630 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java @@ -26,11 +26,9 @@ public class NativeQueryRunnerUtils { private NativeQueryRunnerUtils() {} - public static Map getNativeWorkerHiveProperties(String storageFormat) + public static Map getNativeWorkerHiveProperties() { - return ImmutableMap.of("hive.storage-format", storageFormat, - "hive.pushdown-filter-enabled", "true", - "hive.parquet.pushdown-filter-enabled", "true", + return ImmutableMap.of("hive.parquet.pushdown-filter-enabled", "true", "hive.orc-compression-codec", "ZSTD"); } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index acd0244e054f5..5a40dc5ad1591 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -44,7 +44,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -60,7 +60,6 @@ import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.getNativeSidecarProperties; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.getNativeWorkerHiveProperties; -import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.getNativeWorkerIcebergProperties; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.getNativeWorkerSystemProperties; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.getNativeWorkerTpcdsProperties; import static com.facebook.presto.nativeworker.SymlinkManifestGeneratorUtils.createSymlinkManifest; @@ -71,6 +70,11 @@ public class PrestoNativeQueryRunnerUtils { + public enum QueryRunnerType { + JAVA, + NATIVE + } + // The unix domain socket (UDS) used to communicate with the remote function server. public static final String REMOTE_FUNCTION_UDS = "remote_function_server.socket"; public static final String REMOTE_FUNCTION_JSON_SIGNATURES = "remote_function_server.json"; @@ -92,406 +96,289 @@ public class PrestoNativeQueryRunnerUtils private PrestoNativeQueryRunnerUtils() {} - public static QueryRunner createQueryRunner(boolean addStorageFormatToPath, boolean isCoordinatorSidecarEnabled, boolean enableRuntimeMetricsCollection, boolean enableSsdCache) - throws Exception - { - int cacheMaxSize = 4096; // 4GB size cache - NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); - return createQueryRunner( - Optional.of(nativeQueryRunnerParameters.serverBinary.toString()), - Optional.of(nativeQueryRunnerParameters.dataDirectory), - nativeQueryRunnerParameters.workerCount, - cacheMaxSize, - DEFAULT_STORAGE_FORMAT, - addStorageFormatToPath, - isCoordinatorSidecarEnabled, - enableRuntimeMetricsCollection, - enableSsdCache); - } - - public static QueryRunner createQueryRunner( - Optional prestoServerPath, - Optional dataDirectory, - Optional workerCount, - int cacheMaxSize, - String storageFormat, - boolean addStorageFormatToPath, - boolean isCoordinatorSidecarEnabled, - boolean enableRuntimeMetricsCollection, - boolean enableSsdCache) - throws Exception - { - QueryRunner defaultQueryRunner = createJavaQueryRunner(dataDirectory, storageFormat, addStorageFormatToPath); - - if (!prestoServerPath.isPresent()) { - return defaultQueryRunner; - } - - defaultQueryRunner.close(); - - return createNativeQueryRunner(dataDirectory.get().toString(), prestoServerPath.get(), workerCount, cacheMaxSize, true, Optional.empty(), - storageFormat, addStorageFormatToPath, false, isCoordinatorSidecarEnabled, false, enableRuntimeMetricsCollection, enableSsdCache, Collections.emptyMap()); - } - - public static QueryRunner createJavaQueryRunner() - throws Exception + public static HiveQueryRunnerBuilder nativeHiveQueryRunnerBuilder() { - return createJavaQueryRunner(true); + return new HiveQueryRunnerBuilder(QueryRunnerType.NATIVE); } - public static QueryRunner createJavaQueryRunner(boolean addStorageFormatToPath) - throws Exception + public static HiveQueryRunnerBuilder javaHiveQueryRunnerBuilder() { - return createJavaQueryRunner(DEFAULT_STORAGE_FORMAT, addStorageFormatToPath); + return new HiveQueryRunnerBuilder(QueryRunnerType.JAVA); } - public static QueryRunner createJavaQueryRunner(String storageFormat) - throws Exception + public static class HiveQueryRunnerBuilder { - return createJavaQueryRunner(Optional.of(getNativeQueryRunnerParameters().dataDirectory), storageFormat, true); - } - - public static QueryRunner createJavaQueryRunner(String storageFormat, boolean addStorageFormatToPath) - throws Exception - { - return createJavaQueryRunner(Optional.of(getNativeQueryRunnerParameters().dataDirectory), storageFormat, addStorageFormatToPath); - } + private NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); + private Path dataDirectory = nativeQueryRunnerParameters.dataDirectory; + private String serverBinary = nativeQueryRunnerParameters.serverBinary.toString(); + private Integer workerCount = nativeQueryRunnerParameters.workerCount.orElse(4); + private Integer cacheMaxSize = 0; + private String storageFormat = DEFAULT_STORAGE_FORMAT; + private Optional remoteFunctionServerUds = Optional.empty(); + private Map extraProperties = new HashMap<>(); + private Map extraCoordinatorProperties = new HashMap<>(); + private Map hiveProperties = new HashMap<>(); + private Map tpcdsProperties = new HashMap<>(); + private String security; + private boolean addStorageFormatToPath; + private boolean coordinatorSidecarEnabled; + private boolean enableRuntimeMetricsCollection; + private boolean enableSsdCache; + private boolean failOnNestedLoopJoin; + // External worker launcher is applicable only for the native hive query runner, since it depends on other + // properties it should be created once all the other query runner configs are set. This variable indicates + // whether the query runner returned by builder should use an external worker launcher, it will be true only + // for the native query runner and should NOT be explicitly configured by users. + private boolean useExternalWorkerLauncher; - public static QueryRunner createJavaQueryRunner(Optional dataDirectory, String storageFormat, boolean addStorageFormatToPath) - throws Exception - { - return createJavaQueryRunner(dataDirectory, "sql-standard", storageFormat, addStorageFormatToPath); - } - - public static QueryRunner createJavaQueryRunner(Optional baseDataDirectory, String security, String storageFormat, boolean addStorageFormatToPath) - throws Exception - { - ImmutableMap.Builder hivePropertiesBuilder = new ImmutableMap.Builder<>(); - hivePropertiesBuilder - .put("hive.storage-format", storageFormat) - .put("hive.pushdown-filter-enabled", "true"); - - if ("legacy".equals(security)) { - hivePropertiesBuilder.put("hive.allow-drop-table", "true"); + private HiveQueryRunnerBuilder(QueryRunnerType queryRunnerType) + { + this.hiveProperties.putAll(ImmutableMap.builder() + .put("hive.storage-format", this.storageFormat) + .put("hive.pushdown-filter-enabled", "true") + .build()); + this.tpcdsProperties.putAll(getNativeWorkerTpcdsProperties()); + + if (queryRunnerType.equals(QueryRunnerType.NATIVE)) { + this.extraProperties.putAll(getNativeWorkerSystemProperties()); + this.extraCoordinatorProperties.putAll(ImmutableMap.builder() + .put("native-execution-enabled", "true") + .put("http-server.http.port", "8081") + .build()); + // The property "hive.allow-drop-table" needs to be set to true because security is always "legacy" in NativeQueryRunner. + this.hiveProperties.putAll(ImmutableMap.builder() + .putAll(getNativeWorkerHiveProperties()) + .put("hive.allow-drop-table", "true") + .build()); + this.security = "legacy"; + this.useExternalWorkerLauncher = true; + } + else { + this.extraProperties.putAll(ImmutableMap.of( + "regex-library", "RE2J", + "offset-clause-enabled", "true")); + this.security = "sql-standard"; + this.useExternalWorkerLauncher = false; + } } - Optional dataDirectory = addStorageFormatToPath ? baseDataDirectory.map(path -> Paths.get(path.toString() + '/' + storageFormat)) : baseDataDirectory; - DistributedQueryRunner queryRunner = - HiveQueryRunner.createQueryRunner( - ImmutableList.of(), - ImmutableMap.of( - "regex-library", "RE2J", - "offset-clause-enabled", "true"), - security, - hivePropertiesBuilder.build(), - dataDirectory, - getNativeWorkerTpcdsProperties()); - return queryRunner; - } + public HiveQueryRunnerBuilder setRemoteFunctionServerUds(Optional remoteFunctionServerUds) + { + this.remoteFunctionServerUds = remoteFunctionServerUds; + return this; + } - public static void createSchemaIfNotExist(QueryRunner queryRunner, String schemaName) - { - ExtendedHiveMetastore metastore = getFileHiveMetastore((DistributedQueryRunner) queryRunner); - if (!metastore.getDatabase(METASTORE_CONTEXT, schemaName).isPresent()) { - metastore.createDatabase(METASTORE_CONTEXT, createDatabaseMetastoreObject(schemaName)); + public HiveQueryRunnerBuilder setFailOnNestedLoopJoin(boolean failOnNestedLoopJoin) + { + this.failOnNestedLoopJoin = failOnNestedLoopJoin; + return this; } - } - public static void createExternalTable(QueryRunner queryRunner, String sourceSchemaName, String tableName, List columns, String targetSchemaName) - { - ExtendedHiveMetastore metastore = getFileHiveMetastore((DistributedQueryRunner) queryRunner); - File dataDirectory = ((DistributedQueryRunner) queryRunner).getCoordinator().getDataDirectory().resolve(HIVE_DATA).toFile(); - Path hiveTableDataPath = dataDirectory.toPath().resolve(sourceSchemaName).resolve(tableName); - Path symlinkTableDataPath = dataDirectory.toPath().getParent().resolve(SYMLINK_FOLDER).resolve(tableName); + public HiveQueryRunnerBuilder setUseThrift(boolean useThrift) + { + this.extraProperties + .put("experimental.internal-communication.thrift-transport-enabled", String.valueOf(useThrift)); + return this; + } - try { - createSymlinkManifest(hiveTableDataPath, symlinkTableDataPath); + public HiveQueryRunnerBuilder setSingleNodeExecutionEnabled(boolean singleNodeExecutionEnabled) + { + if (singleNodeExecutionEnabled) { + this.extraCoordinatorProperties.put("single-node-execution-enabled", "true"); + } + return this; } - catch (IOException e) { - throw new PrestoException(() -> CREATE_ERROR_CODE, "Failed to create symlink manifest file for table: " + tableName, e); + + public HiveQueryRunnerBuilder setEnableSsdCache(boolean enableSsdCache) + { + this.enableSsdCache = enableSsdCache; + return this; } - createSchemaIfNotExist(queryRunner, targetSchemaName); - if (!metastore.getTable(METASTORE_CONTEXT, targetSchemaName, tableName).isPresent()) { - metastore.createTable(METASTORE_CONTEXT, createHiveSymlinkTable(targetSchemaName, tableName, columns, symlinkTableDataPath.toString()), PRINCIPAL_PRIVILEGES, emptyList()); + public HiveQueryRunnerBuilder setEnableRuntimeMetricsCollection(boolean enableRuntimeMetricsCollection) + { + this.enableRuntimeMetricsCollection = enableRuntimeMetricsCollection; + return this; } - } - public static QueryRunner createJavaIcebergQueryRunner(boolean addStorageFormatToPath) - throws Exception - { - return createJavaIcebergQueryRunner(Optional.of(getNativeQueryRunnerParameters().dataDirectory), ICEBERG_DEFAULT_STORAGE_FORMAT, addStorageFormatToPath); - } + public HiveQueryRunnerBuilder setCoordinatorSidecarEnabled(boolean coordinatorSidecarEnabled) + { + this.coordinatorSidecarEnabled = coordinatorSidecarEnabled; + if (coordinatorSidecarEnabled) { + this.extraProperties.putAll(getNativeSidecarProperties()); + } + return this; + } - public static QueryRunner createJavaIcebergQueryRunner(String storageFormat) - throws Exception - { - return createJavaIcebergQueryRunner(Optional.of(getNativeQueryRunnerParameters().dataDirectory), storageFormat, false); - } + public HiveQueryRunnerBuilder setStorageFormat(String storageFormat) + { + this.storageFormat = storageFormat; + this.hiveProperties.put("hive.storage-format", storageFormat); + return this; + } - public static QueryRunner createJavaIcebergQueryRunner(Optional baseDataDirectory, String storageFormat, boolean addStorageFormatToPath) - throws Exception - { - ImmutableMap.Builder icebergPropertiesBuilder = new ImmutableMap.Builder<>(); - icebergPropertiesBuilder.put("hive.parquet.writer.version", "PARQUET_1_0"); + public HiveQueryRunnerBuilder setAddStorageFormatToPath(boolean addStorageFormatToPath) + { + this.addStorageFormatToPath = addStorageFormatToPath; + return this; + } - return IcebergQueryRunner.builder() - .setExtraProperties(ImmutableMap.of( - "regex-library", "RE2J", - "offset-clause-enabled", "true", - "query.max-stage-count", "110")) - .setExtraConnectorProperties(icebergPropertiesBuilder.build()) - .setAddJmxPlugin(false) - .setCreateTpchTables(false) - .setDataDirectory(baseDataDirectory) - .setAddStorageFormatToPath(addStorageFormatToPath) - .setFormat(FileFormat.valueOf(storageFormat)) - .setTpcdsProperties(getNativeWorkerTpcdsProperties()) - .build().getQueryRunner(); - } + public HiveQueryRunnerBuilder setCacheMaxSize(Integer cacheMaxSize) + { + this.cacheMaxSize = cacheMaxSize; + return this; + } - public static QueryRunner createNativeIcebergQueryRunner(boolean useThrift) - throws Exception - { - return createNativeIcebergQueryRunner(useThrift, ICEBERG_DEFAULT_STORAGE_FORMAT, Optional.empty()); - } + public HiveQueryRunnerBuilder setExtraProperties(Map extraProperties) + { + this.extraProperties.putAll(extraProperties); + return this; + } - public static QueryRunner createNativeIcebergQueryRunner(boolean useThrift, boolean addStorageFormatToPath) - throws Exception - { - return createNativeIcebergQueryRunner(useThrift, ICEBERG_DEFAULT_STORAGE_FORMAT, Optional.empty(), addStorageFormatToPath); - } + public HiveQueryRunnerBuilder setHiveProperties(Map hiveProperties) + { + this.hiveProperties.putAll(hiveProperties); + return this; + } - public static QueryRunner createNativeIcebergQueryRunner(boolean useThrift, String storageFormat) - throws Exception - { - return createNativeIcebergQueryRunner(useThrift, storageFormat, Optional.empty()); + public QueryRunner build() + throws Exception + { + Optional> externalWorkerLauncher = Optional.empty(); + if (this.useExternalWorkerLauncher) { + externalWorkerLauncher = getExternalWorkerLauncher("hive", serverBinary, cacheMaxSize, remoteFunctionServerUds, + failOnNestedLoopJoin, coordinatorSidecarEnabled, enableRuntimeMetricsCollection, enableSsdCache); + } + return HiveQueryRunner.createQueryRunner( + ImmutableList.of(), + ImmutableList.of(), + extraProperties, + extraCoordinatorProperties, + security, + hiveProperties, + Optional.ofNullable(workerCount), + Optional.of(Paths.get(addStorageFormatToPath ? dataDirectory.toString() + "/" + storageFormat : dataDirectory.toString())), + externalWorkerLauncher, + tpcdsProperties); + } } - public static QueryRunner createNativeIcebergQueryRunner(boolean useThrift, String storageFormat, Optional remoteFunctionServerUds) - throws Exception + public static IcebergQueryRunnerBuilder nativeIcebergQueryRunnerBuilder() { - return createNativeIcebergQueryRunner(useThrift, storageFormat, remoteFunctionServerUds, false); + return new IcebergQueryRunnerBuilder(QueryRunnerType.NATIVE); } - public static QueryRunner createNativeIcebergQueryRunner(boolean useThrift, String storageFormat, Optional remoteFunctionServerUds, boolean addStorageFormatToPath) - throws Exception + public static IcebergQueryRunnerBuilder javaIcebergQueryRunnerBuilder() { - int cacheMaxSize = 0; - NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); - return createNativeIcebergQueryRunner( - Optional.of(nativeQueryRunnerParameters.dataDirectory), - nativeQueryRunnerParameters.serverBinary.toString(), - nativeQueryRunnerParameters.workerCount, - cacheMaxSize, - useThrift, - remoteFunctionServerUds, - storageFormat, - addStorageFormatToPath); + return new IcebergQueryRunnerBuilder(QueryRunnerType.JAVA); } - public static QueryRunner createNativeIcebergQueryRunner( - Optional dataDirectory, - String prestoServerPath, - Optional workerCount, - int cacheMaxSize, - boolean useThrift, - Optional remoteFunctionServerUds, - String storageFormat, - boolean addStorageFormatToPath) - throws Exception + public static class IcebergQueryRunnerBuilder { - ImmutableMap icebergProperties = ImmutableMap.builder() - .putAll(getNativeWorkerIcebergProperties()) - .build(); + private NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); + private Path dataDirectory = nativeQueryRunnerParameters.dataDirectory; + private String serverBinary = nativeQueryRunnerParameters.serverBinary.toString(); + private Integer workerCount = nativeQueryRunnerParameters.workerCount.orElse(4); + private Integer cacheMaxSize = 0; + private String storageFormat = ICEBERG_DEFAULT_STORAGE_FORMAT; + private Map extraProperties = new HashMap<>(); + private Map extraConnectorProperties = new HashMap<>(); + private Optional remoteFunctionServerUds = Optional.empty(); + private boolean addStorageFormatToPath; + // External worker launcher is applicable only for the native iceberg query runner, since it depends on other + // properties it should be created once all the other query runner configs are set. This variable indicates + // whether the query runner returned by builder should use an external worker launcher, it will be true only + // for the native query runner and should NOT be explicitly configured by users. + private boolean useExternalWorkerLauncher; - // Make query runner with external workers for tests - return IcebergQueryRunner.builder() - .setExtraProperties(ImmutableMap.builder() + private IcebergQueryRunnerBuilder(QueryRunnerType queryRunnerType) + { + if (queryRunnerType.equals(QueryRunnerType.NATIVE)) { + this.extraProperties.putAll(ImmutableMap.builder() .put("http-server.http.port", "8080") - .put("experimental.internal-communication.thrift-transport-enabled", String.valueOf(useThrift)) .put("query.max-stage-count", "110") .putAll(getNativeWorkerSystemProperties()) - .build()) - .setFormat(FileFormat.valueOf(storageFormat)) - .setCreateTpchTables(false) - .setAddJmxPlugin(false) - .setNodeCount(OptionalInt.of(workerCount.orElse(4))) - .setExternalWorkerLauncher(getExternalWorkerLauncher("iceberg", prestoServerPath, cacheMaxSize, remoteFunctionServerUds, false, false, false, false)) - .setAddStorageFormatToPath(addStorageFormatToPath) - .setDataDirectory(dataDirectory) - .setTpcdsProperties(getNativeWorkerTpcdsProperties()) - .build().getQueryRunner(); - } - - public static QueryRunner createNativeQueryRunner( - String dataDirectory, - String prestoServerPath, - Optional workerCount, - int cacheMaxSize, - boolean useThrift, - Optional remoteFunctionServerUds, - String storageFormat, - boolean addStorageFormatToPath, - Boolean failOnNestedLoopJoin, - boolean isCoordinatorSidecarEnabled, - boolean singleNodeExecutionEnabled, - boolean enableRuntimeMetricsCollection, - boolean enableSsdCache, - Map extraProperties) - throws Exception - { - // The property "hive.allow-drop-table" needs to be set to true because security is always "legacy" in NativeQueryRunner. - ImmutableMap hiveProperties = ImmutableMap.builder() - .putAll(getNativeWorkerHiveProperties(storageFormat)) - .put("hive.allow-drop-table", "true") - .build(); - - ImmutableMap.Builder coordinatorProperties = ImmutableMap.builder(); - coordinatorProperties.put("native-execution-enabled", "true"); - if (singleNodeExecutionEnabled) { - coordinatorProperties.put("single-node-execution-enabled", "true"); + .build()); + this.useExternalWorkerLauncher = true; + } + else { + this.extraProperties.putAll(ImmutableMap.of( + "regex-library", "RE2J", + "offset-clause-enabled", "true", + "query.max-stage-count", "110")); + this.extraConnectorProperties.putAll(ImmutableMap.of("hive.parquet.writer.version", "PARQUET_1_0")); + this.useExternalWorkerLauncher = false; + } } - // Make query runner with external workers for tests - return HiveQueryRunner.createQueryRunner( - ImmutableList.of(), - ImmutableList.of(), - ImmutableMap.builder() - .put("http-server.http.port", "8081") - .put("experimental.internal-communication.thrift-transport-enabled", String.valueOf(useThrift)) - .putAll(getNativeWorkerSystemProperties()) - .putAll(isCoordinatorSidecarEnabled ? getNativeSidecarProperties() : ImmutableMap.of()) - .putAll(extraProperties) - .build(), - coordinatorProperties.build(), - "legacy", - hiveProperties, - workerCount, - Optional.of(Paths.get(addStorageFormatToPath ? dataDirectory + "/" + storageFormat : dataDirectory)), - getExternalWorkerLauncher("hive", prestoServerPath, cacheMaxSize, remoteFunctionServerUds, failOnNestedLoopJoin, - isCoordinatorSidecarEnabled, enableRuntimeMetricsCollection, enableSsdCache), - getNativeWorkerTpcdsProperties()); - } - - public static QueryRunner createNativeCteQueryRunner(boolean useThrift, String storageFormat) - throws Exception - { - return createNativeCteQueryRunner(useThrift, storageFormat, true); - } + public IcebergQueryRunnerBuilder setStorageFormat(String storageFormat) + { + this.storageFormat = storageFormat; + return this; + } - public static QueryRunner createNativeCteQueryRunner(boolean useThrift, String storageFormat, boolean addStorageFormatToPath) - throws Exception - { - int cacheMaxSize = 0; - - NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); - String dataDirectory = nativeQueryRunnerParameters.dataDirectory.toString(); - String prestoServerPath = nativeQueryRunnerParameters.serverBinary.toString(); - Optional workerCount = nativeQueryRunnerParameters.workerCount; - - // The property "hive.allow-drop-table" needs to be set to true because security is always "legacy" in NativeQueryRunner. - ImmutableMap hiveProperties = ImmutableMap.builder() - .putAll(getNativeWorkerHiveProperties(storageFormat)) - .put("hive.allow-drop-table", "true") - .put("hive.enable-parquet-dereference-pushdown", "true") - .put("hive.temporary-table-compression-codec", "NONE") - .put("hive.temporary-table-storage-format", storageFormat) - .build(); - - // Make query runner with external workers for tests - return HiveQueryRunner.createQueryRunner( - ImmutableList.of(), - ImmutableList.of(), - ImmutableMap.builder() - .put("http-server.http.port", "8081") - .put("experimental.internal-communication.thrift-transport-enabled", String.valueOf(useThrift)) - .putAll(getNativeWorkerSystemProperties()) - .put("query.cte-partitioning-provider-catalog", "hive") - .build(), - ImmutableMap.of(), - "legacy", - hiveProperties, - workerCount, - Optional.of(Paths.get(addStorageFormatToPath ? dataDirectory + "/" + storageFormat : dataDirectory)), - getExternalWorkerLauncher("hive", prestoServerPath, cacheMaxSize, Optional.empty(), false, false, false, false), - getNativeWorkerTpcdsProperties()); - } + public IcebergQueryRunnerBuilder setAddStorageFormatToPath(boolean addStorageFormatToPath) + { + this.addStorageFormatToPath = addStorageFormatToPath; + return this; + } - public static QueryRunner createNativeQueryRunner(String remoteFunctionServerUds) - throws Exception - { - return createNativeQueryRunner(false, DEFAULT_STORAGE_FORMAT, Optional.ofNullable(remoteFunctionServerUds), false, false, false, false, false); - } + public IcebergQueryRunnerBuilder setUseThrift(boolean useThrift) + { + this.extraProperties + .put("experimental.internal-communication.thrift-transport-enabled", String.valueOf(useThrift)); + return this; + } - public static QueryRunner createNativeQueryRunner(Map extraProperties, String storageFormat) - throws Exception - { - int cacheMaxSize = 0; - NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); - return createNativeQueryRunner( - nativeQueryRunnerParameters.dataDirectory.toString(), - nativeQueryRunnerParameters.serverBinary.toString(), - nativeQueryRunnerParameters.workerCount, - cacheMaxSize, - true, - Optional.empty(), - storageFormat, - true, - false, - false, - false, - false, - false, - extraProperties); + public QueryRunner build() + throws Exception + { + Optional> externalWorkerLauncher = Optional.empty(); + if (this.useExternalWorkerLauncher) { + externalWorkerLauncher = getExternalWorkerLauncher("iceberg", serverBinary, cacheMaxSize, remoteFunctionServerUds, + false, false, false, false); + } + return IcebergQueryRunner.builder() + .setExtraProperties(extraProperties) + .setExtraConnectorProperties(extraConnectorProperties) + .setFormat(FileFormat.valueOf(storageFormat)) + .setCreateTpchTables(false) + .setAddJmxPlugin(false) + .setNodeCount(OptionalInt.of(workerCount)) + .setExternalWorkerLauncher(externalWorkerLauncher) + .setAddStorageFormatToPath(addStorageFormatToPath) + .setDataDirectory(Optional.of(dataDirectory)) + .setTpcdsProperties(getNativeWorkerTpcdsProperties()) + .build().getQueryRunner(); + } } - public static QueryRunner createNativeQueryRunner(boolean useThrift) - throws Exception + public static void createSchemaIfNotExist(QueryRunner queryRunner, String schemaName) { - return createNativeQueryRunner(useThrift, DEFAULT_STORAGE_FORMAT); + ExtendedHiveMetastore metastore = getFileHiveMetastore((DistributedQueryRunner) queryRunner); + if (!metastore.getDatabase(METASTORE_CONTEXT, schemaName).isPresent()) { + metastore.createDatabase(METASTORE_CONTEXT, createDatabaseMetastoreObject(schemaName)); + } } - public static QueryRunner createNativeQueryRunner(boolean useThrift, boolean failOnNestedLoopJoin) - throws Exception + public static void createExternalTable(QueryRunner queryRunner, String sourceSchemaName, String tableName, List columns, String targetSchemaName) { - return createNativeQueryRunner(useThrift, DEFAULT_STORAGE_FORMAT, Optional.empty(), failOnNestedLoopJoin, false, false, false, false); - } + ExtendedHiveMetastore metastore = getFileHiveMetastore((DistributedQueryRunner) queryRunner); + File dataDirectory = ((DistributedQueryRunner) queryRunner).getCoordinator().getDataDirectory().resolve(HIVE_DATA).toFile(); + Path hiveTableDataPath = dataDirectory.toPath().resolve(sourceSchemaName).resolve(tableName); + Path symlinkTableDataPath = dataDirectory.toPath().getParent().resolve(SYMLINK_FOLDER).resolve(tableName); - public static QueryRunner createNativeQueryRunner(boolean useThrift, String storageFormat) - throws Exception - { - return createNativeQueryRunner(useThrift, storageFormat, Optional.empty(), false, false, false, false, false); - } + try { + createSymlinkManifest(hiveTableDataPath, symlinkTableDataPath); + } + catch (IOException e) { + throw new PrestoException(() -> CREATE_ERROR_CODE, "Failed to create symlink manifest file for table: " + tableName, e); + } - public static QueryRunner createNativeQueryRunner( - boolean useThrift, - String storageFormat, - Optional remoteFunctionServerUds, - Boolean failOnNestedLoopJoin, - boolean isCoordinatorSidecarEnabled, - boolean singleNodeExecutionEnabled, - boolean enableRuntimeMetricsCollection, - boolean enableSSDCache) - throws Exception - { - int cacheMaxSize = 0; - NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); - return createNativeQueryRunner( - nativeQueryRunnerParameters.dataDirectory.toString(), - nativeQueryRunnerParameters.serverBinary.toString(), - nativeQueryRunnerParameters.workerCount, - cacheMaxSize, - useThrift, - remoteFunctionServerUds, - storageFormat, - true, - failOnNestedLoopJoin, - isCoordinatorSidecarEnabled, - singleNodeExecutionEnabled, - enableRuntimeMetricsCollection, - enableSSDCache, - Collections.emptyMap()); + createSchemaIfNotExist(queryRunner, targetSchemaName); + if (!metastore.getTable(METASTORE_CONTEXT, targetSchemaName, tableName).isPresent()) { + metastore.createTable(METASTORE_CONTEXT, createHiveSymlinkTable(targetSchemaName, tableName, columns, symlinkTableDataPath.toString()), PRINCIPAL_PRIVILEGES, emptyList()); + } } // Start the remote function server. Return the UDS path used to communicate with it. diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativeHistoryBasedStatsTracking.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativeHistoryBasedStatsTracking.java index e13968371d964..17b78529bae2f 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativeHistoryBasedStatsTracking.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativeHistoryBasedStatsTracking.java @@ -37,7 +37,10 @@ public class TestNativeHistoryBasedStatsTracking protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @BeforeMethod(alwaysRun = true) diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativePlanValidation.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativePlanValidation.java index 707d336caf595..e503698956699 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativePlanValidation.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestNativePlanValidation.java @@ -19,6 +19,9 @@ public class TestNativePlanValidation { protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false, true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setFailOnNestedLoopJoin(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAggregations.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAggregations.java index 04facd9c3384d..38dca52bb69bf 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAggregations.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAggregations.java @@ -22,12 +22,16 @@ public class TestPrestoNativeAggregations @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java index 92f60c3e746e4..bfa21641206b2 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java @@ -22,12 +22,16 @@ public class TestPrestoNativeArrayFunctionQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java index 5161190f191cb..8c2b251215126 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java @@ -43,13 +43,21 @@ public class TestPrestoNativeAsyncDataCacheCleanupAPI @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createQueryRunner(true, false, true, true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setCacheMaxSize(4096) + .setUseThrift(true) + .setAddStorageFormatToPath(true) + .setEnableRuntimeMetricsCollection(true) + .setEnableSsdCache(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBitwiseFunctionQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBitwiseFunctionQueries.java index 3459fc5149c6c..3b0918014474b 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBitwiseFunctionQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBitwiseFunctionQueries.java @@ -22,12 +22,16 @@ public class TestPrestoNativeBitwiseFunctionQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCtasQueriesParquet.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCtasQueriesParquet.java index 228f649ac7c7f..5b8ceade2b7c5 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCtasQueriesParquet.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCtasQueriesParquet.java @@ -21,17 +21,26 @@ public class TestPrestoNativeCtasQueriesParquet extends AbstractTestNativeCtasQueries { + private final String storageFormat = "PARQUET"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCteExecutionParquet.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCteExecutionParquet.java index 05b460e3a50d6..a360eaac82711 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCteExecutionParquet.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeCteExecutionParquet.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; import static com.facebook.presto.SystemSessionProperties.CTE_FILTER_AND_PROJECTION_PUSHDOWN_ENABLED; @@ -27,18 +28,33 @@ public class TestPrestoNativeCteExecutionParquet extends AbstractTestNativeCteExecution { + private final String storageFormat = "PARQUET"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeCteQueryRunner(true, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .setExtraProperties(ImmutableMap.of("query.cte-partitioning-provider-catalog", "hive")) + .setHiveProperties(ImmutableMap.builder() + .put("hive.enable-parquet-dereference-pushdown", "true") + .put("hive.temporary-table-compression-codec", "NONE") + .put("hive.temporary-table-storage-format", this.storageFormat) + .build()) + .build(); } @Override protected QueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .setAddStorageFormatToPath(true) + .build(); } @Override diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java index a26963d12fe19..4b1d27f30426e 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java @@ -23,13 +23,17 @@ public class TestPrestoNativeGeneralQueriesJSON protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java index 7a1af31a2b35b..a1c33cd09f51c 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java @@ -23,13 +23,18 @@ public class TestPrestoNativeGeneralQueriesThrift protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGracefulShutdown.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGracefulShutdown.java index cdc45fdf13b77..c066615178475 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGracefulShutdown.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGracefulShutdown.java @@ -14,7 +14,6 @@ package com.facebook.presto.nativeworker; import com.facebook.presto.spi.NodeState; -import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.DistributedQueryRunner; import org.testng.annotations.Test; @@ -26,8 +25,10 @@ public class TestPrestoNativeGracefulShutdown @Test public void testGracefulShutdown() throws Exception { - QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); - DistributedQueryRunner distributedQueryRunner = (DistributedQueryRunner) queryRunner; + DistributedQueryRunner distributedQueryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); int responseCode = distributedQueryRunner.sendWorkerRequest(0, "INVALID_BODY"); assertEquals(responseCode, 400, "Expected a 400 Bad Request response for invalid body"); @@ -40,6 +41,6 @@ public void testGracefulShutdown() throws Exception NodeState state = distributedQueryRunner.getWorkerInfoState(0); assertEquals(state.getValue(), SHUTTING_DOWN.getValue()); - queryRunner.close(); + distributedQueryRunner.close(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHistoryBasedStatistics.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHistoryBasedStatistics.java index 9e1ccd5007212..f1fc79c186720 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHistoryBasedStatistics.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHistoryBasedStatistics.java @@ -55,7 +55,10 @@ protected void createTables() protected QueryRunner createQueryRunner() throws Exception { - QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); queryRunner.installPlugin(new Plugin() { @Override @@ -71,7 +74,9 @@ public Iterable getHistoryBasedPlanStatistic protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @BeforeMethod(alwaysRun = true) diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHiveExternalTableTpchQueriesParquet.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHiveExternalTableTpchQueriesParquet.java index 37184cbd6c448..d5841ff11fc5f 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHiveExternalTableTpchQueriesParquet.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeHiveExternalTableTpchQueriesParquet.java @@ -21,15 +21,24 @@ public class TestPrestoNativeHiveExternalTableTpchQueriesParquet extends AbstractTestNativeHiveExternalTableTpchQueries { + private final String storageFormat = "PARQUET"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergGeneralQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergGeneralQueries.java index 59ce312ce4584..d653f514550f4 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergGeneralQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergGeneralQueries.java @@ -18,6 +18,7 @@ import com.facebook.presto.tests.AbstractTestQueryFramework; import org.testng.annotations.Test; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.ICEBERG_DEFAULT_STORAGE_FORMAT; import static java.lang.String.format; import static org.testng.Assert.assertEquals; @@ -28,14 +29,20 @@ public class TestPrestoNativeIcebergGeneralQueries protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(false, true); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner(true); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setAddStorageFormatToPath(true) + .build(); } @Override diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift.java index c18eb82b444d4..1b56ca3c5eb16 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift.java @@ -16,9 +16,6 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; -import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner; -import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner; - public class TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift extends AbstractTestNativeIcebergPositionDeleteQueries { @@ -28,13 +25,18 @@ public class TestPrestoNativeIcebergPositionDeleteQueriesOrcUsingThrift protected QueryRunner createQueryRunner() throws Exception { - return createNativeIcebergQueryRunner(true, storageFormat); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return createJavaIcebergQueryRunner(storageFormat); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift.java index c6515fb4a18f9..923a4ba474932 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift.java @@ -16,9 +16,6 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; -import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner; -import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner; - public class TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift extends AbstractTestNativeIcebergPositionDeleteQueries { @@ -28,13 +25,18 @@ public class TestPrestoNativeIcebergPositionDeleteQueriesParquetUsingThrift protected QueryRunner createQueryRunner() throws Exception { - return createNativeIcebergQueryRunner(true, storageFormat); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return createJavaIcebergQueryRunner(storageFormat); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesOrcUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesOrcUsingThrift.java index 3754e66aa8ea7..0205ba49ba171 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesOrcUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesOrcUsingThrift.java @@ -26,7 +26,10 @@ protected QueryRunner createQueryRunner() throws Exception { this.storageFormat = "ORC"; - return PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(true, "ORC"); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .setUseThrift(true) + .build(); } @Override @@ -34,7 +37,9 @@ protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { this.storageFormat = "ORC"; - return PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner("ORC"); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .build(); } @Test diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesParquetUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesParquetUsingThrift.java index 297cc05ce6dd9..0e9d76207860a 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesParquetUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpcdsQueriesParquetUsingThrift.java @@ -26,7 +26,10 @@ protected QueryRunner createQueryRunner() throws Exception { this.storageFormat = "PARQUET"; - return PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(true, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .setUseThrift(true) + .build(); } @Override @@ -34,7 +37,9 @@ protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { this.storageFormat = "PARQUET"; - return PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .build(); } @Test diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesOrcUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesOrcUsingThrift.java index 23499fd39d414..c5ba89202c6c9 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesOrcUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesOrcUsingThrift.java @@ -24,12 +24,17 @@ public class TestPrestoNativeIcebergTpchQueriesOrcUsingThrift @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(true, storageFormat); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner(storageFormat); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesParquetUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesParquetUsingThrift.java index ce94fe3bc3213..9cea2f2e2806f 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesParquetUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTpchQueriesParquetUsingThrift.java @@ -19,15 +19,22 @@ public class TestPrestoNativeIcebergTpchQueriesParquetUsingThrift extends AbstractTestNativeIcebergTpchQueries { + private final String storageFormat = "PARQUET"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeIcebergQueryRunner(true, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaIcebergQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeJoinQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeJoinQueries.java index 922bf17b26819..d45c4f1256335 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeJoinQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeJoinQueries.java @@ -22,12 +22,17 @@ public class TestPrestoNativeJoinQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeProbabilityFunctionQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeProbabilityFunctionQueries.java index 64caf6c4d889b..67e56089670e7 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeProbabilityFunctionQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeProbabilityFunctionQueries.java @@ -21,12 +21,16 @@ public class TestPrestoNativeProbabilityFunctionQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected QueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeRemoteFunctions.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeRemoteFunctions.java index f93f4c1e303eb..6dfc656ce35b0 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeRemoteFunctions.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeRemoteFunctions.java @@ -16,18 +16,24 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; +import java.util.Optional; + public class TestPrestoNativeRemoteFunctions extends AbstractTestNativeRemoteFunctions { @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(remoteFunctionServerUds); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setRemoteFunctionServerUds(Optional.of(remoteFunctionServerUds)) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueries.java index 68fe0e87a779f..062915fa066c0 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueries.java @@ -23,13 +23,17 @@ public class TestPrestoNativeSystemQueries protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueriesSingleNode.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueriesSingleNode.java index e6657d81c8677..b61274c7382e4 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueriesSingleNode.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeSystemQueriesSingleNode.java @@ -16,30 +16,30 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; -import java.util.Optional; - public class TestPrestoNativeSystemQueriesSingleNode extends AbstractTestNativeSystemQueries { + private final String storageFormat = "DWRF"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( - true, - "DWRF", - Optional.empty(), - false, - false, - true, - false, - false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .setSingleNodeExecutionEnabled(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java index 828f378971cd8..dfa8a91db721b 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java @@ -25,7 +25,11 @@ public class TestPrestoNativeTpcdsQueriesOrcUsingThrift protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, "ORC"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat("ORC") + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override @@ -33,6 +37,9 @@ protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { this.storageFormat = "ORC"; - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("ORC"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesParquetUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesParquetUsingThrift.java index 369599942b76a..4f51893976acf 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesParquetUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesParquetUsingThrift.java @@ -25,7 +25,11 @@ public class TestPrestoNativeTpcdsQueriesParquetUsingThrift protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat("PARQUET") + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override @@ -33,6 +37,9 @@ protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { this.storageFormat = "PARQUET"; - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(this.storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchConnectorQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchConnectorQueries.java index 71fd7c2c7c12e..ad382c2b2eb54 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchConnectorQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchConnectorQueries.java @@ -23,14 +23,19 @@ public class TestPrestoNativeTpchConnectorQueries protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesDwrfUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesDwrfUsingThrift.java index 2cf6068e861c4..9c373a7b84446 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesDwrfUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesDwrfUsingThrift.java @@ -19,15 +19,24 @@ public class TestPrestoNativeTpchQueriesDwrfUsingThrift extends AbstractTestNativeTpchQueries { + private final String storageFormat = "DWRF"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, "DWRF"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("DWRF"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingJSON.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingJSON.java index 141c94d610065..9244bb8df45b3 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingJSON.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingJSON.java @@ -21,15 +21,23 @@ public class TestPrestoNativeTpchQueriesOrcUsingJSON extends AbstractTestNativeTpchQueries { + private final String storageFormat = "ORC"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false, "ORC"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("ORC"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingThrift.java index 43a77a0cc55aa..fee30303ff8f3 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesOrcUsingThrift.java @@ -19,15 +19,24 @@ public class TestPrestoNativeTpchQueriesOrcUsingThrift extends AbstractTestNativeTpchQueries { + private final String storageFormat = "ORC"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true, "ORC"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("ORC"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesParquetUsingJSON.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesParquetUsingJSON.java index 81f7cc8fa4fb6..a0b10936bfda7 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesParquetUsingJSON.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpchQueriesParquetUsingJSON.java @@ -21,15 +21,23 @@ public class TestPrestoNativeTpchQueriesParquetUsingJSON extends AbstractTestNativeTpchQueries { + private final String storageFormat = "PARQUET"; + @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false, "PARQUET"); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner("PARQUET"); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java index 5e61f97a788f6..908fe3ea0c605 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java @@ -22,12 +22,17 @@ public class TestPrestoNativeWindowQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(true); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWriter.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWriter.java index c9dfd52e1c34d..1a9b4a2cb40a3 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWriter.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWriter.java @@ -70,13 +70,17 @@ public class TestPrestoNativeWriter @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(false); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Override diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunner.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunner.java index cbfcda2bf1eb2..ead98f4a62f43 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunner.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunner.java @@ -33,12 +33,16 @@ public static void main(String[] args) Logging.initialize(); // Create tables before launching distributed runner. - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(false); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(false) + .build(); NativeQueryRunnerUtils.createAllTables(javaQueryRunner); javaQueryRunner.close(); // Launch distributed runner. - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(false, true, false, false); + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setCoordinatorSidecarEnabled(true) + .build(); setupNativeSidecarPlugin(queryRunner); Thread.sleep(10); Logger log = Logger.get(DistributedQueryRunner.class); diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java index 7bab4909d10a7..6c54207a48c39 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java @@ -61,7 +61,10 @@ protected void createTables() protected QueryRunner createQueryRunner() throws Exception { - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(true, true, false, false); + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .setCoordinatorSidecarEnabled(true) + .build(); setupNativeSidecarPlugin(queryRunner); return queryRunner; } @@ -70,7 +73,9 @@ protected QueryRunner createQueryRunner() protected QueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); } @Test diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginWithoutLoadingFunctionalities.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginWithoutLoadingFunctionalities.java index 607d81c81c312..01e786a4a0f66 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginWithoutLoadingFunctionalities.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginWithoutLoadingFunctionalities.java @@ -39,7 +39,9 @@ protected void createTables() protected QueryRunner createQueryRunner() throws Exception { - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(true, false, false, false); + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); // Installing the native sidecar plugin on a native cluster does not load the plugin functionalities because // we aren't loading the individual functionalities. queryRunner.installCoordinatorPlugin(new NativeSidecarPlugin()); @@ -50,7 +52,9 @@ protected QueryRunner createQueryRunner() protected QueryRunner createExpectedQueryRunner() throws Exception { - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); // Installing the native sidecar plugin on a Java cluster, does not load the plugin functionalities because // we aren't loading the individual functionalities. queryRunner.installCoordinatorPlugin(new NativeSidecarPlugin()); diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java index e0ae26b31fd65..9f60d93f30312 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java @@ -17,7 +17,6 @@ import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestEngineOnlyQueries; -import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; import org.testng.annotations.Parameters; import org.testng.annotations.Test; @@ -39,7 +38,11 @@ public class TestDistributedEngineOnlyQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(System.getProperty("storageFormat")) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Parameters("storageFormat") @@ -48,7 +51,10 @@ protected void createTables() { try { String storageFormat = System.getProperty("storageFormat"); - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); if (storageFormat.equals("DWRF")) { NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOrderByQueries.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOrderByQueries.java index ec8f32b2261fd..529d18e55ca6f 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOrderByQueries.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOrderByQueries.java @@ -17,7 +17,6 @@ import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestOrderByQueries; -import com.google.common.collect.ImmutableMap; import org.testng.annotations.Parameters; import org.testng.annotations.Test; @@ -28,7 +27,11 @@ public class TestOrderByQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(System.getProperty("storageFormat")) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Parameters("storageFormat") @@ -37,7 +40,10 @@ protected void createTables() { try { String storageFormat = System.getProperty("storageFormat"); - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); if (storageFormat.equals("DWRF")) { NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueries.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueries.java index ebc8f19105c09..898649862193f 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueries.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueries.java @@ -17,7 +17,6 @@ import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestRepartitionQueries; -import com.google.common.collect.ImmutableMap; import org.testng.annotations.Parameters; public class TestRepartitionQueries @@ -27,7 +26,11 @@ public class TestRepartitionQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(System.getProperty("storageFormat")) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Parameters("storageFormat") @@ -36,7 +39,10 @@ protected void createTables() { try { String storageFormat = System.getProperty("storageFormat"); - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); if (storageFormat.equals("DWRF")) { NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueriesWithSmallPages.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueriesWithSmallPages.java index 5e354f3cd3cf6..14a0102edb249 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueriesWithSmallPages.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestRepartitionQueriesWithSmallPages.java @@ -27,10 +27,14 @@ public class TestRepartitionQueriesWithSmallPages @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner( - // Use small SerializedPages to force flushing - ImmutableMap.of("driver.max-page-partitioning-buffer-size", "200B"), - System.getProperty("storageFormat")); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(System.getProperty("storageFormat")) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .setExtraProperties( + // Use small SerializedPages to force flushing + ImmutableMap.of("driver.max-page-partitioning-buffer-size", "200B")) + .build(); } @Parameters("storageFormat") @@ -39,7 +43,10 @@ protected void createTables() { try { String storageFormat = System.getProperty("storageFormat"); - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); if (storageFormat.equals("DWRF")) { NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestWindowQueries.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestWindowQueries.java index 6f173e80419f8..86f40db18744e 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestWindowQueries.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestWindowQueries.java @@ -17,7 +17,6 @@ import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestWindowQueries; -import com.google.common.collect.ImmutableMap; import org.testng.annotations.Parameters; import org.testng.annotations.Test; @@ -30,7 +29,11 @@ public class TestWindowQueries @Override protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.createNativeQueryRunner(ImmutableMap.of(), System.getProperty("storageFormat")); + return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(System.getProperty("storageFormat")) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .build(); } @Parameters("storageFormat") @@ -39,7 +42,10 @@ protected void createTables() { try { String storageFormat = System.getProperty("storageFormat"); - QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.createJavaQueryRunner(storageFormat); + QueryRunner javaQueryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .build(); if (storageFormat.equals("DWRF")) { NativeQueryRunnerUtils.createAllTables(javaQueryRunner, true); }