From 852e865aecfb0f0c31ebcbfae72c0474a94cf0a8 Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Fri, 29 Nov 2019 09:26:02 +0100 Subject: [PATCH 1/2] Rename metrics enable/disable configs --- .../org/apache/spark/internal/config/Status.scala | 2 +- .../org/apache/spark/internal/config/package.scala | 2 +- docs/monitoring.md | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/Status.scala b/core/src/main/scala/org/apache/spark/internal/config/Status.scala index 3e6a4e981066..fd9ff071ffc6 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Status.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Status.scala @@ -56,7 +56,7 @@ private[spark] object Status { .createWithDefault(Int.MaxValue) val APP_STATUS_METRICS_ENABLED = - ConfigBuilder("spark.app.status.metrics.enabled") + ConfigBuilder("spark.metrics.appStatusSource.enabled") .doc("Whether Dropwizard/Codahale metrics " + "will be reported for the status of the running spark app.") .booleanConf diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 0f402e0bf2bb..5448ffd537b4 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -638,7 +638,7 @@ package object config { .createOptional private[spark] val METRICS_STATIC_SOURCES_ENABLED = - ConfigBuilder("spark.metrics.static.sources.enabled") + ConfigBuilder("spark.metrics.staticSources.enabled") .doc("Whether to register static sources with the metrics system.") .booleanConf .createWithDefault(true) diff --git a/docs/monitoring.md b/docs/monitoring.md index f094d0ba0dfc..2aed452adb05 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -924,7 +924,7 @@ This is the component with the largest amount of instrumented metrics - namespace=HiveExternalCatalog - **note:**: these metrics are conditional to a configuration parameter: - `spark.metrics.static.sources.enabled` (default is true) + `spark.metrics.staticSources.enabled` (default is true) - fileCacheHits.count - filesDiscovered.count - hiveClientCalls.count @@ -933,7 +933,7 @@ This is the component with the largest amount of instrumented metrics - namespace=CodeGenerator - **note:**: these metrics are conditional to a configuration parameter: - `spark.metrics.static.sources.enabled` (default is true) + `spark.metrics.staticSources.enabled` (default is true) - compilationTime (histogram) - generatedClassSize (histogram) - generatedMethodSize (histogram) @@ -962,8 +962,8 @@ This is the component with the largest amount of instrumented metrics - queue.executorManagement.listenerProcessingTime (timer) - namespace=appStatus (all metrics of type=counter) - - **note:** Introduced in Spark 3.0. Conditional to configuration parameter: - `spark.app.status.metrics.enabled=true` (default is false) + - **note:** Introduced in Spark 3.0. Conditional to a configuration parameter: + `spark.metrics.appStatusSource.enabled` (default is false) - stages.failedStages.count - stages.skippedStages.count - stages.completedStages.count @@ -1057,7 +1057,7 @@ when running in local mode. - namespace=HiveExternalCatalog - **note:**: these metrics are conditional to a configuration parameter: - `spark.metrics.static.sources.enabled` (default is true) + `spark.metrics.staticSources.enabled` (default is true) - fileCacheHits.count - filesDiscovered.count - hiveClientCalls.count @@ -1066,7 +1066,7 @@ when running in local mode. - namespace=CodeGenerator - **note:**: these metrics are conditional to a configuration parameter: - `spark.metrics.static.sources.enabled` (default is true) + `spark.metrics.staticSources.enabled` (default is true) - compilationTime (histogram) - generatedClassSize (histogram) - generatedMethodSize (histogram) From 49052de53438341b7d1669fcb52a3126e1422157 Mon Sep 17 00:00:00 2001 From: Luca Canali Date: Tue, 3 Dec 2019 08:37:58 +0100 Subject: [PATCH 2/2] Renamed APP_STATUS_METRICS_ENABLED to METRICS_APP_STATUS_SOURCE_ENABLED --- .../main/scala/org/apache/spark/internal/config/Status.scala | 2 +- .../main/scala/org/apache/spark/status/AppStatusSource.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/Status.scala b/core/src/main/scala/org/apache/spark/internal/config/Status.scala index fd9ff071ffc6..3cc00a6f094c 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Status.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Status.scala @@ -55,7 +55,7 @@ private[spark] object Status { .intConf .createWithDefault(Int.MaxValue) - val APP_STATUS_METRICS_ENABLED = + val METRICS_APP_STATUS_SOURCE_ENABLED = ConfigBuilder("spark.metrics.appStatusSource.enabled") .doc("Whether Dropwizard/Codahale metrics " + "will be reported for the status of the running spark app.") diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala b/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala index f6a21578ff49..20f171bd3c37 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusSource.scala @@ -22,7 +22,7 @@ import AppStatusSource.getCounter import com.codahale.metrics.{Counter, Gauge, MetricRegistry} import org.apache.spark.SparkConf -import org.apache.spark.internal.config.Status.APP_STATUS_METRICS_ENABLED +import org.apache.spark.internal.config.Status.METRICS_APP_STATUS_SOURCE_ENABLED import org.apache.spark.metrics.source.Source private [spark] class JobDuration(val value: AtomicLong) extends Gauge[Long] { @@ -71,7 +71,7 @@ private[spark] object AppStatusSource { } def createSource(conf: SparkConf): Option[AppStatusSource] = { - Option(conf.get(APP_STATUS_METRICS_ENABLED)) + Option(conf.get(METRICS_APP_STATUS_SOURCE_ENABLED)) .filter(identity) .map { _ => new AppStatusSource() } }