From f022816c0882dc99f49d77aee260d87f6ab3e97b Mon Sep 17 00:00:00 2001 From: Zebing Lin Date: Wed, 5 Jul 2023 16:25:17 -0400 Subject: [PATCH 1/2] Only resort/update FTE plan if plan after optimization has changed --- .../scheduler/EventDrivenFaultTolerantQueryScheduler.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/EventDrivenFaultTolerantQueryScheduler.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/EventDrivenFaultTolerantQueryScheduler.java index 59f0d17a5081..863b75195647 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/EventDrivenFaultTolerantQueryScheduler.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/EventDrivenFaultTolerantQueryScheduler.java @@ -758,9 +758,12 @@ public void onFailure(Throwable t) private void optimize() { + SubPlan oldPlan = plan; plan = optimizePlan(plan); - planInTopologicalOrder = sortPlanInTopologicalOrder(plan); - stageRegistry.updatePlan(plan); + if (plan != oldPlan) { + planInTopologicalOrder = sortPlanInTopologicalOrder(plan); + stageRegistry.updatePlan(plan); + } } private SubPlan optimizePlan(SubPlan plan) From 6cb2bda44e17c42b484742f2f05daf1e991eaff0 Mon Sep 17 00:00:00 2001 From: Zebing Lin Date: Wed, 5 Jul 2023 16:27:19 -0400 Subject: [PATCH 2/2] Change adaptive query execution growth factor to be 1.26 After https://github.com/trinodb/trino/pull/17834, 1.26 becomes the better default. --- .../src/main/java/io/trino/execution/QueryManagerConfig.java | 4 ++-- .../test/java/io/trino/execution/TestQueryManagerConfig.java | 4 ++-- docs/src/main/sphinx/admin/fault-tolerant-execution.rst | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java index 5c7bb5d07985..6e562c9d5220 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java @@ -111,12 +111,12 @@ public class QueryManagerConfig private int remoteTaskGuaranteedSplitPerTask = 3; private int faultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeGrowthPeriod = 64; - private double faultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeGrowthFactor = 1.2; + private double faultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeGrowthFactor = 1.26; private DataSize faultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeMin = DataSize.of(512, MEGABYTE); private DataSize faultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeMax = DataSize.of(50, GIGABYTE); private int faultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeGrowthPeriod = 64; - private double faultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeGrowthFactor = 1.2; + private double faultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeGrowthFactor = 1.26; private DataSize faultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeMin = DataSize.of(4, GIGABYTE); private DataSize faultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeMax = DataSize.of(50, GIGABYTE); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java index d356f5842d55..73bd6afd9b4b 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java @@ -80,11 +80,11 @@ public void testDefaults() .setRemoteTaskRequestSizeHeadroom(DataSize.of(2, DataSize.Unit.MEGABYTE)) .setRemoteTaskGuaranteedSplitPerTask(3) .setFaultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeGrowthPeriod(64) - .setFaultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeGrowthFactor(1.2) + .setFaultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeGrowthFactor(1.26) .setFaultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeMin(DataSize.of(512, MEGABYTE)) .setFaultTolerantExecutionArbitraryDistributionComputeTaskTargetSizeMax(DataSize.of(50, GIGABYTE)) .setFaultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeGrowthPeriod(64) - .setFaultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeGrowthFactor(1.2) + .setFaultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeGrowthFactor(1.26) .setFaultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeMin(DataSize.of(4, GIGABYTE)) .setFaultTolerantExecutionArbitraryDistributionWriteTaskTargetSizeMax(DataSize.of(50, GIGABYTE)) .setFaultTolerantExecutionHashDistributionComputeTaskTargetSize(DataSize.of(512, MEGABYTE)) diff --git a/docs/src/main/sphinx/admin/fault-tolerant-execution.rst b/docs/src/main/sphinx/admin/fault-tolerant-execution.rst index 9bb43d910b7d..e1465f1c3ec0 100644 --- a/docs/src/main/sphinx/admin/fault-tolerant-execution.rst +++ b/docs/src/main/sphinx/admin/fault-tolerant-execution.rst @@ -256,7 +256,7 @@ properties only apply to a ``TASK`` retry policy. distribution for fault-tolerant execution. Lower bound is 1.0. For every task size increase, new task target size is old task target size multiplied by this growth factor. - - ``1.2`` + - ``1.26`` * - ``fault-tolerant-execution-arbitrary-distribution-compute-task-target-size-min`` - Initial/minimum target input :ref:`data size ` for non-writer tasks of arbitrary distribution of fault-tolerant execution. @@ -274,7 +274,7 @@ properties only apply to a ``TASK`` retry policy. distribution for fault-tolerant execution. Lower bound is 1.0. For every task size increase, new task target size is old task target size multiplied by this growth factor. - - ``1.2`` + - ``1.26`` * - ``fault-tolerant-execution-arbitrary-distribution-write-task-target-size-min`` - Initial/minimum target input :ref:`data size ` for writer tasks of arbitrary distribution of fault-tolerant execution.