Skip to content

Commit 6466143

Browse files
tanjialiangtanjialiang
committed
Fix spill config plumbing for pos cpp
1 parent 3545b1d commit 6466143

File tree

3 files changed

+91
-92
lines changed

3 files changed

+91
-92
lines changed

‎presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionSystemConfig.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,13 @@ public class NativeExecutionSystemConfig
111111
private static final String HTTP_SERVER_ACCESS_LOGS = "http-server.enable-access-log";
112112
// Terminates the native process and generates a core file on an allocation failure
113113
private static final String CORE_ON_ALLOCATION_FAILURE_ENABLED = "core-on-allocation-failure-enabled";
114+
// Spill related properties
115+
private static final String SPILL_ENABLED = "spill-enabled";
116+
private static final String AGGREGATION_SPILL_ENABLED = "aggregation-spill-enabled";
117+
private static final String JOIN_SPILL_ENABLED = "join-spill-enabled";
118+
private static final String ORDER_BY_SPILL_ENABLED = "order-by-spill-enabled";
119+
private static final String MAX_SPILL_BYTES = "max-spill-bytes";
120+
114121
private boolean enableSerializedPageChecksum = true;
115122
private boolean enableVeloxExpressionLogging;
116123
private boolean enableVeloxTaskLogging = true;
@@ -151,6 +158,11 @@ public class NativeExecutionSystemConfig
151158
private boolean registerTestFunctions;
152159
private boolean enableHttpServerAccessLog = true;
153160
private boolean coreOnAllocationFailureEnabled;
161+
private boolean spillEnabled = true;
162+
private boolean aggregationSpillEnabled = true;
163+
private boolean joinSpillEnabled = true;
164+
private boolean orderBySpillEnabled = true;
165+
private Long maxSpillBytes = 600L << 30;
154166

155167
public Map<String, String> getAllProperties()
156168
{
@@ -191,6 +203,11 @@ public Map<String, String> getAllProperties()
191203
.put(SHUFFLE_NAME, getShuffleName())
192204
.put(HTTP_SERVER_ACCESS_LOGS, String.valueOf(isEnableHttpServerAccessLog()))
193205
.put(CORE_ON_ALLOCATION_FAILURE_ENABLED, String.valueOf(isCoreOnAllocationFailureEnabled()))
206+
.put(SPILL_ENABLED, String.valueOf(getSpillEnabled()))
207+
.put(AGGREGATION_SPILL_ENABLED, String.valueOf(getAggregationSpillEnabled()))
208+
.put(JOIN_SPILL_ENABLED, String.valueOf(getJoinSpillEnabled()))
209+
.put(ORDER_BY_SPILL_ENABLED, String.valueOf(getOrderBySpillEnabled()))
210+
.put(MAX_SPILL_BYTES, String.valueOf(getMaxSpillBytes()))
194211
.build();
195212
}
196213

@@ -625,4 +642,64 @@ public NativeExecutionSystemConfig setCoreOnAllocationFailureEnabled(boolean cor
625642
this.coreOnAllocationFailureEnabled = coreOnAllocationFailureEnabled;
626643
return this;
627644
}
645+
646+
public boolean getSpillEnabled()
647+
{
648+
return spillEnabled;
649+
}
650+
651+
@Config(SPILL_ENABLED)
652+
public NativeExecutionSystemConfig setSpillEnabled(boolean spillEnabled)
653+
{
654+
this.spillEnabled = spillEnabled;
655+
return this;
656+
}
657+
658+
public boolean getAggregationSpillEnabled()
659+
{
660+
return aggregationSpillEnabled;
661+
}
662+
663+
@Config(AGGREGATION_SPILL_ENABLED)
664+
public NativeExecutionSystemConfig setAggregationSpillEnabled(boolean aggregationSpillEnabled)
665+
{
666+
this.aggregationSpillEnabled = aggregationSpillEnabled;
667+
return this;
668+
}
669+
670+
public boolean getJoinSpillEnabled()
671+
{
672+
return joinSpillEnabled;
673+
}
674+
675+
@Config(JOIN_SPILL_ENABLED)
676+
public NativeExecutionSystemConfig setJoinSpillEnabled(boolean joinSpillEnabled)
677+
{
678+
this.joinSpillEnabled = joinSpillEnabled;
679+
return this;
680+
}
681+
682+
public boolean getOrderBySpillEnabled()
683+
{
684+
return orderBySpillEnabled;
685+
}
686+
687+
@Config(ORDER_BY_SPILL_ENABLED)
688+
public NativeExecutionSystemConfig setOrderBySpillEnabled(boolean orderBySpillEnabled)
689+
{
690+
this.orderBySpillEnabled = orderBySpillEnabled;
691+
return this;
692+
}
693+
694+
public Long getMaxSpillBytes()
695+
{
696+
return maxSpillBytes;
697+
}
698+
699+
@Config(MAX_SPILL_BYTES)
700+
public NativeExecutionSystemConfig setMaxSpillBytes(Long maxSpillBytes)
701+
{
702+
this.maxSpillBytes = maxSpillBytes;
703+
return this;
704+
}
628705
}

‎presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionVeloxConfig.java‎

Lines changed: 0 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -24,31 +24,13 @@
2424
public class NativeExecutionVeloxConfig
2525
{
2626
private static final String CODEGEN_ENABLED = "codegen.enabled";
27-
// Spilling related configs.
28-
private static final String SPILL_ENABLED = "spill_enabled";
29-
private static final String AGGREGATION_SPILL_ENABLED = "aggregation_spill_enabled";
30-
private static final String JOIN_SPILL_ENABLED = "join_spill_enabled";
31-
private static final String ORDER_BY_SPILL_ENABLED = "order_by_spill_enabled";
32-
private static final String MAX_SPILL_BYTES = "max_spill_bytes";
3327

3428
private boolean codegenEnabled;
35-
private boolean spillEnabled = true;
36-
private boolean aggregationSpillEnabled = true;
37-
private boolean joinSpillEnabled = true;
38-
private boolean orderBySpillEnabled = true;
39-
// Velox default value is 100GB, as it is designed for Presto cluster
40-
// use-case. But for presto-on-spark, 500GB is a reasonable default
41-
private Long maxSpillBytes = 500L << 30;
4229

4330
public Map<String, String> getAllProperties()
4431
{
4532
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
4633
return builder.put(CODEGEN_ENABLED, String.valueOf(getCodegenEnabled()))
47-
.put(SPILL_ENABLED, String.valueOf(getSpillEnabled()))
48-
.put(AGGREGATION_SPILL_ENABLED, String.valueOf(getAggregationSpillEnabled()))
49-
.put(JOIN_SPILL_ENABLED, String.valueOf(getJoinSpillEnabled()))
50-
.put(ORDER_BY_SPILL_ENABLED, String.valueOf(getOrderBySpillEnabled()))
51-
.put(MAX_SPILL_BYTES, String.valueOf(getMaxSpillBytes()))
5234
.build();
5335
}
5436

@@ -63,64 +45,4 @@ public NativeExecutionVeloxConfig setCodegenEnabled(boolean codegenEnabled)
6345
this.codegenEnabled = codegenEnabled;
6446
return this;
6547
}
66-
67-
public boolean getSpillEnabled()
68-
{
69-
return spillEnabled;
70-
}
71-
72-
@Config(SPILL_ENABLED)
73-
public NativeExecutionVeloxConfig setSpillEnabled(boolean spillEnabled)
74-
{
75-
this.spillEnabled = spillEnabled;
76-
return this;
77-
}
78-
79-
public boolean getAggregationSpillEnabled()
80-
{
81-
return aggregationSpillEnabled;
82-
}
83-
84-
@Config(AGGREGATION_SPILL_ENABLED)
85-
public NativeExecutionVeloxConfig setAggregationSpillEnabled(boolean aggregationSpillEnabled)
86-
{
87-
this.aggregationSpillEnabled = aggregationSpillEnabled;
88-
return this;
89-
}
90-
91-
public boolean getJoinSpillEnabled()
92-
{
93-
return joinSpillEnabled;
94-
}
95-
96-
@Config(JOIN_SPILL_ENABLED)
97-
public NativeExecutionVeloxConfig setJoinSpillEnabled(boolean joinSpillEnabled)
98-
{
99-
this.joinSpillEnabled = joinSpillEnabled;
100-
return this;
101-
}
102-
103-
public boolean getOrderBySpillEnabled()
104-
{
105-
return orderBySpillEnabled;
106-
}
107-
108-
@Config(ORDER_BY_SPILL_ENABLED)
109-
public NativeExecutionVeloxConfig setOrderBySpillEnabled(boolean orderBySpillEnabled)
110-
{
111-
this.orderBySpillEnabled = orderBySpillEnabled;
112-
return this;
113-
}
114-
115-
public Long getMaxSpillBytes()
116-
{
117-
return maxSpillBytes;
118-
}
119-
120-
@Config(MAX_SPILL_BYTES)
121-
public NativeExecutionVeloxConfig setMaxSpillBytes(Long maxSpillBytes)
122-
{
123-
this.maxSpillBytes = maxSpillBytes;
124-
return this;
125-
}
12648
}

‎presto-spark-base/src/test/java/com/facebook/presto/spark/execution/property/TestNativeExecutionSystemConfig.java‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,21 +39,11 @@ public void testNativeExecutionVeloxConfig()
3939
{
4040
// Test defaults
4141
assertRecordedDefaults(ConfigAssertions.recordDefaults(NativeExecutionVeloxConfig.class)
42-
.setCodegenEnabled(false)
43-
.setSpillEnabled(true)
44-
.setAggregationSpillEnabled(true)
45-
.setJoinSpillEnabled(true)
46-
.setOrderBySpillEnabled(true)
47-
.setMaxSpillBytes(500L << 30));
42+
.setCodegenEnabled(false));
4843

4944
// Test explicit property mapping. Also makes sure properties returned by getAllProperties() covers full property list.
5045
NativeExecutionVeloxConfig expected = new NativeExecutionVeloxConfig()
51-
.setCodegenEnabled(true)
52-
.setSpillEnabled(false)
53-
.setAggregationSpillEnabled(false)
54-
.setJoinSpillEnabled(false)
55-
.setOrderBySpillEnabled(false)
56-
.setMaxSpillBytes(1L);
46+
.setCodegenEnabled(true);
5747
Map<String, String> properties = expected.getAllProperties();
5848
assertFullMapping(properties, expected);
5949
}
@@ -98,7 +88,12 @@ public void testNativeExecutionSystemConfig()
9888
.setShuffleName("local")
9989
.setRegisterTestFunctions(false)
10090
.setEnableHttpServerAccessLog(true)
101-
.setCoreOnAllocationFailureEnabled(false));
91+
.setCoreOnAllocationFailureEnabled(false)
92+
.setSpillEnabled(true)
93+
.setAggregationSpillEnabled(true)
94+
.setJoinSpillEnabled(true)
95+
.setOrderBySpillEnabled(true)
96+
.setMaxSpillBytes(600L << 30));
10297

10398
// Test explicit property mapping. Also makes sure properties returned by getAllProperties() covers full property list.
10499
NativeExecutionSystemConfig expected = new NativeExecutionSystemConfig()
@@ -137,7 +132,12 @@ public void testNativeExecutionSystemConfig()
137132
.setShuffleName("custom")
138133
.setRegisterTestFunctions(true)
139134
.setEnableHttpServerAccessLog(false)
140-
.setCoreOnAllocationFailureEnabled(true);
135+
.setCoreOnAllocationFailureEnabled(true)
136+
.setSpillEnabled(false)
137+
.setAggregationSpillEnabled(false)
138+
.setJoinSpillEnabled(false)
139+
.setOrderBySpillEnabled(false)
140+
.setMaxSpillBytes(1L);
141141
Map<String, String> properties = expected.getAllProperties();
142142
assertFullMapping(properties, expected);
143143
}

0 commit comments

Comments
 (0)