diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala index 84bd430d9abe..24c987187049 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala @@ -74,7 +74,7 @@ private[v1] class StagesResource extends BaseAppResource { def taskSummary( @PathParam("stageId") stageId: Int, @PathParam("stageAttemptId") stageAttemptId: Int, - @DefaultValue("0.05,0.25,0.5,0.75,0.95") @QueryParam("quantiles") quantileString: String) + @DefaultValue("0,0.25,0.5,0.75,1.0") @QueryParam("quantiles") quantileString: String) : TaskMetricDistributions = withUI { ui => val quantiles = quantileString.split(",").map { s => try { diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json index e6b705989cc9..ab59545e87cd 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json @@ -1,5 +1,5 @@ { - "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ], + "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ], "executorDeserializeTime" : [ 1.0, 2.0, 2.0, 2.0, 3.0 ], "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "executorRunTime" : [ 30.0, 74.0, 75.0, 76.0, 79.0 ], diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json index 788f28cf7b36..1b074c2bfece 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json @@ -1,19 +1,19 @@ { - "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ], - "executorDeserializeTime" : [ 2.0, 2.0, 3.0, 7.0, 31.0 ], + "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ], + "executorDeserializeTime" : [ 1.0, 2.0, 3.0, 7.0, 36.0 ], "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], - "executorRunTime" : [ 16.0, 18.0, 28.0, 49.0, 349.0 ], + "executorRunTime" : [ 14.0, 18.0, 28.0, 49.0, 351.0 ], "executorCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "resultSize" : [ 2010.0, 2065.0, 2065.0, 2065.0, 2065.0 ], "jvmGcTime" : [ 0.0, 0.0, 0.0, 5.0, 7.0 ], - "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 1.0 ], + "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 2.0 ], "gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], - "schedulerDelay" : [ 2.0, 4.0, 6.0, 13.0, 40.0 ], + "schedulerDelay" : [ 2.0, 4.0, 6.0, 13.0, 53.0 ], "peakExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "inputMetrics" : { - "bytesRead" : [ 60488.0, 70564.0, 70564.0, 70564.0, 70564.0 ], + "bytesRead" : [ 49294.0, 70564.0, 70564.0, 70564.0, 70565.0 ], "recordsRead" : [ 10000.0, 10000.0, 10000.0, 10000.0, 10000.0 ] }, "outputMetrics" : { @@ -33,6 +33,6 @@ "shuffleWriteMetrics" : { "writeBytes" : [ 1710.0, 1710.0, 1710.0, 1710.0, 1710.0 ], "writeRecords" : [ 10.0, 10.0, 10.0, 10.0, 10.0 ], - "writeTime" : [ 90329.0, 95848.0, 102159.0, 121551.0, 2579051.0 ] + "writeTime" : [ 83022.0, 95848.0, 102159.0, 121551.0, 4016617.0 ] } } diff --git a/docs/monitoring.md b/docs/monitoring.md index 5b3278bca031..a5e5244fa8c1 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -490,6 +490,7 @@ can be identified by their `[attempt-id]`. In the API listed below, when running Summary metrics of all tasks in the given stage attempt.
?quantiles summarize the metrics with the given quantiles. +
The default value is 0.0,0.25,0.5,0.75,1.0.
Example: ?quantiles=0.01,0.5,0.99