diff --git a/receiver/apachesparkreceiver/documentation.md b/receiver/apachesparkreceiver/documentation.md index 290fc7e6389b7..fb94c1382f2db 100644 --- a/receiver/apachesparkreceiver/documentation.md +++ b/receiver/apachesparkreceiver/documentation.md @@ -16,17 +16,17 @@ metrics: Disk space used by the BlockManager. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| mb | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| mb | Sum | Int | Cumulative | false | development | ### spark.driver.block_manager.memory.usage Memory usage for the driver's BlockManager. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| mb | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| mb | Sum | Int | Cumulative | false | development | #### Attributes @@ -39,89 +39,89 @@ Memory usage for the driver's BlockManager. Average time spent during CodeGenerator source code compilation operations. -| Unit | Metric Type | Value Type | -| ---- | ----------- | ---------- | -| ms | Gauge | Double | +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| ms | Gauge | Double | development | ### spark.driver.code_generator.compilation.count Number of source code compilation operations performed by the CodeGenerator. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { compilation } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { compilation } | Sum | Int | Cumulative | true | development | ### spark.driver.code_generator.generated_class.average_size Average class size of the classes generated by the CodeGenerator. -| Unit | Metric Type | Value Type | -| ---- | ----------- | ---------- | -| bytes | Gauge | Double | +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| bytes | Gauge | Double | development | ### spark.driver.code_generator.generated_class.count Number of classes generated by the CodeGenerator. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { class } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { class } | Sum | Int | Cumulative | true | development | ### spark.driver.code_generator.generated_method.average_size Average method size of the classes generated by the CodeGenerator. -| Unit | Metric Type | Value Type | -| ---- | ----------- | ---------- | -| bytes | Gauge | Double | +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| bytes | Gauge | Double | development | ### spark.driver.code_generator.generated_method.count Number of methods generated by the CodeGenerator. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { method } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { method } | Sum | Int | Cumulative | true | development | ### spark.driver.code_generator.source_code.average_size Average size of the source code generated by a CodeGenerator code generation operation. -| Unit | Metric Type | Value Type | -| ---- | ----------- | ---------- | -| bytes | Gauge | Double | +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| bytes | Gauge | Double | development | ### spark.driver.code_generator.source_code.operations Number of source code generation operations performed by the CodeGenerator. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { operation } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { operation } | Sum | Int | Cumulative | true | development | ### spark.driver.dag_scheduler.job.active Number of active jobs currently being processed by the DAGScheduler. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { job } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { job } | Sum | Int | Cumulative | false | development | ### spark.driver.dag_scheduler.job.count Number of jobs that have been submitted to the DAGScheduler. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { job } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { job } | Sum | Int | Cumulative | true | development | ### spark.driver.dag_scheduler.stage.count Number of stages the DAGScheduler is either running or needs to run. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { stage } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { stage } | Sum | Int | Cumulative | false | development | #### Attributes @@ -133,17 +133,17 @@ Number of stages the DAGScheduler is either running or needs to run. Number of failed stages run by the DAGScheduler. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { stage } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { stage } | Sum | Int | Cumulative | true | development | ### spark.driver.executor.gc.operations Number of garbage collection operations performed by the driver. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { gc_operation } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { gc_operation } | Sum | Int | Cumulative | true | development | #### Attributes @@ -155,9 +155,9 @@ Number of garbage collection operations performed by the driver. Total elapsed time during garbage collection operations performed by the driver. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ms | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ms | Sum | Int | Cumulative | true | development | #### Attributes @@ -169,9 +169,9 @@ Total elapsed time during garbage collection operations performed by the driver. Amount of execution memory currently used by the driver. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | #### Attributes @@ -183,9 +183,9 @@ Amount of execution memory currently used by the driver. Amount of memory used by the driver's JVM. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | #### Attributes @@ -197,9 +197,9 @@ Amount of memory used by the driver's JVM. Amount of pool memory currently used by the driver. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | #### Attributes @@ -211,9 +211,9 @@ Amount of pool memory currently used by the driver. Amount of storage memory currently used by the driver. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | #### Attributes @@ -225,121 +225,121 @@ Amount of storage memory currently used by the driver. Number of file cache hits on the HiveExternalCatalog. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { hit } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { hit } | Sum | Int | Cumulative | true | development | ### spark.driver.hive_external_catalog.files_discovered Number of files discovered while listing the partitions of a table in the Hive metastore -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { file } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { file } | Sum | Int | Cumulative | true | development | ### spark.driver.hive_external_catalog.hive_client_calls Number of calls to the underlying Hive Metastore client made by the Spark application. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { call } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { call } | Sum | Int | Cumulative | true | development | ### spark.driver.hive_external_catalog.parallel_listing_jobs Number of parallel listing jobs initiated by the HiveExternalCatalog when listing partitions of a table. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { listing_job } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { listing_job } | Sum | Int | Cumulative | true | development | ### spark.driver.hive_external_catalog.partitions_fetched Table partitions fetched by the HiveExternalCatalog. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { partition } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { partition } | Sum | Int | Cumulative | true | development | ### spark.driver.jvm_cpu_time Current CPU time taken by the Spark driver. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ns | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ns | Sum | Int | Cumulative | true | development | ### spark.driver.live_listener_bus.dropped Number of events that have been dropped by the LiveListenerBus. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { event } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { event } | Sum | Int | Cumulative | true | development | ### spark.driver.live_listener_bus.posted Number of events that have been posted on the LiveListenerBus. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { event } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { event } | Sum | Int | Cumulative | true | development | ### spark.driver.live_listener_bus.processing_time.average Average time taken for the LiveListenerBus to process an event posted to it. -| Unit | Metric Type | Value Type | -| ---- | ----------- | ---------- | -| ms | Gauge | Double | +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| ms | Gauge | Double | development | ### spark.driver.live_listener_bus.queue_size Number of events currently waiting to be processed by the LiveListenerBus. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { event } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { event } | Sum | Int | Cumulative | false | development | ### spark.executor.disk.usage Disk space used by this executor for RDD storage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | ### spark.executor.gc_time Elapsed time the JVM spent in garbage collection in this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ms | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ms | Sum | Int | Cumulative | true | development | ### spark.executor.input_size Amount of data input for this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ### spark.executor.memory.usage Storage memory used by this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | ### spark.executor.shuffle.io.size Amount of data written and read during shuffle operations for this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | #### Attributes @@ -351,9 +351,9 @@ Amount of data written and read during shuffle operations for this executor. The executor's storage memory usage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | false | development | #### Attributes @@ -366,25 +366,25 @@ The executor's storage memory usage. Number of tasks currently running in this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | false | development | ### spark.executor.task.limit Maximum number of tasks that can run concurrently in this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | false | development | ### spark.executor.task.result Number of tasks with a specific result in this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | true | development | #### Attributes @@ -396,25 +396,25 @@ Number of tasks with a specific result in this executor. Elapsed time the JVM spent executing tasks in this executor. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ms | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ms | Sum | Int | Cumulative | true | development | ### spark.job.stage.active Number of active stages in this job. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { stage } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { stage } | Sum | Int | Cumulative | false | development | ### spark.job.stage.result Number of stages with a specific result in this job. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { stage } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { stage } | Sum | Int | Cumulative | true | development | #### Attributes @@ -426,17 +426,17 @@ Number of stages with a specific result in this job. Number of active tasks in this job. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | false | development | ### spark.job.task.result Number of tasks with a specific result in this job. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | true | development | #### Attributes @@ -448,33 +448,33 @@ Number of tasks with a specific result in this job. The amount of disk space used for storing portions of overly large data chunks that couldn't fit in memory in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ### spark.stage.executor.cpu_time CPU time spent by the executor in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ns | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ns | Sum | Int | Cumulative | true | development | ### spark.stage.executor.run_time Amount of time spent by the executor in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ms | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ms | Sum | Int | Cumulative | true | development | ### spark.stage.io.records Number of records written and read in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { record } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { record } | Sum | Int | Cumulative | true | development | #### Attributes @@ -486,9 +486,9 @@ Number of records written and read in this stage. Amount of data written and read at this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | #### Attributes @@ -500,33 +500,33 @@ Amount of data written and read at this stage. The amount of time the JVM spent on garbage collection in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ms | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ms | Sum | Int | Cumulative | true | development | ### spark.stage.memory.peak Peak memory used by internal data structures created during shuffles, aggregations and joins in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ### spark.stage.memory.spilled The amount of memory moved to disk due to size constraints (spilled) in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ### spark.stage.shuffle.blocks_fetched Number of blocks fetched in shuffle operations in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { block } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { block } | Sum | Int | Cumulative | true | development | #### Attributes @@ -538,25 +538,25 @@ Number of blocks fetched in shuffle operations in this stage. Time spent in this stage waiting for remote shuffle blocks. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ms | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ms | Sum | Int | Cumulative | true | development | ### spark.stage.shuffle.io.disk Amount of data read to disk in shuffle operations (sometimes required for large blocks, as opposed to the default behavior of reading into memory). -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ### spark.stage.shuffle.io.read.size Amount of data read in shuffle operations in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | #### Attributes @@ -568,9 +568,9 @@ Amount of data read in shuffle operations in this stage. Number of records written or read in shuffle operations in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { record } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { record } | Sum | Int | Cumulative | true | development | #### Attributes @@ -582,25 +582,25 @@ Number of records written or read in shuffle operations in this stage. Amount of data written in shuffle operations in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ### spark.stage.shuffle.write_time Time spent blocking on writes to disk or buffer cache in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| ns | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| ns | Sum | Int | Cumulative | true | development | ### spark.stage.status A one-hot encoding representing the status of this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { status } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { status } | Sum | Int | Cumulative | false | development | #### Attributes @@ -615,17 +615,17 @@ A one-hot encoding representing the status of this stage. Number of active tasks in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | false | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | false | development | ### spark.stage.task.result Number of tasks with a specific result in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| { task } | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| { task } | Sum | Int | Cumulative | true | development | #### Attributes @@ -637,9 +637,9 @@ Number of tasks with a specific result in this stage. The amount of data transmitted back to the driver by all the tasks in this stage. -| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | -| ---- | ----------- | ---------- | ----------------------- | --------- | -| bytes | Sum | Int | Cumulative | true | +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | +| bytes | Sum | Int | Cumulative | true | development | ## Resource Attributes diff --git a/receiver/apachesparkreceiver/metadata.yaml b/receiver/apachesparkreceiver/metadata.yaml index ebb034bdf94fb..fd207bfce80da 100644 --- a/receiver/apachesparkreceiver/metadata.yaml +++ b/receiver/apachesparkreceiver/metadata.yaml @@ -124,6 +124,8 @@ metrics: spark.stage.status: description: A one-hot encoding representing the status of this stage. enabled: true + stability: + level: development sum: monotonic: false aggregation_temporality: cumulative @@ -134,6 +136,8 @@ metrics: spark.stage.task.active: description: Number of active tasks in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -143,6 +147,8 @@ metrics: spark.stage.task.result: description: Number of tasks with a specific result in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -153,6 +159,8 @@ metrics: spark.stage.executor.run_time: description: Amount of time spent by the executor in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -162,6 +170,8 @@ metrics: spark.stage.executor.cpu_time: description: CPU time spent by the executor in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -171,6 +181,8 @@ metrics: spark.stage.task.result_size: description: The amount of data transmitted back to the driver by all the tasks in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -180,6 +192,8 @@ metrics: spark.stage.jvm_gc_time: description: The amount of time the JVM spent on garbage collection in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -189,6 +203,8 @@ metrics: spark.stage.memory.spilled: description: The amount of memory moved to disk due to size constraints (spilled) in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -198,6 +214,8 @@ metrics: spark.stage.disk.spilled: description: The amount of disk space used for storing portions of overly large data chunks that couldn't fit in memory in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -207,6 +225,8 @@ metrics: spark.stage.memory.peak: description: Peak memory used by internal data structures created during shuffles, aggregations and joins in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -216,6 +236,8 @@ metrics: spark.stage.io.size: description: Amount of data written and read at this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -226,6 +248,8 @@ metrics: spark.stage.io.records: description: Number of records written and read in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -236,6 +260,8 @@ metrics: spark.stage.shuffle.blocks_fetched: description: Number of blocks fetched in shuffle operations in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -246,6 +272,8 @@ metrics: spark.stage.shuffle.fetch_wait_time: description: Time spent in this stage waiting for remote shuffle blocks. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -255,6 +283,8 @@ metrics: spark.stage.shuffle.io.disk: description: Amount of data read to disk in shuffle operations (sometimes required for large blocks, as opposed to the default behavior of reading into memory). enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -264,6 +294,8 @@ metrics: spark.stage.shuffle.io.read.size: description: Amount of data read in shuffle operations in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -274,6 +306,8 @@ metrics: spark.stage.shuffle.io.write.size: description: Amount of data written in shuffle operations in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -284,6 +318,8 @@ metrics: spark.stage.shuffle.io.records: description: Number of records written or read in shuffle operations in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -294,6 +330,8 @@ metrics: spark.stage.shuffle.write_time: description: Time spent blocking on writes to disk or buffer cache in this stage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -304,6 +342,8 @@ metrics: spark.executor.memory.usage: description: Storage memory used by this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -313,6 +353,8 @@ metrics: spark.executor.disk.usage: description: Disk space used by this executor for RDD storage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -322,6 +364,8 @@ metrics: spark.executor.task.limit: description: Maximum number of tasks that can run concurrently in this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -331,6 +375,8 @@ metrics: spark.executor.task.active: description: Number of tasks currently running in this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -340,6 +386,8 @@ metrics: spark.executor.task.result: description: Number of tasks with a specific result in this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -349,6 +397,8 @@ metrics: spark.executor.time: description: Elapsed time the JVM spent executing tasks in this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -358,6 +408,8 @@ metrics: spark.executor.gc_time: description: Elapsed time the JVM spent in garbage collection in this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -367,6 +419,8 @@ metrics: spark.executor.input_size: description: Amount of data input for this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -376,6 +430,8 @@ metrics: spark.executor.shuffle.io.size: description: Amount of data written and read during shuffle operations for this executor. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -385,6 +441,8 @@ metrics: spark.executor.storage_memory.usage: description: The executor's storage memory usage. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -395,6 +453,8 @@ metrics: spark.job.task.active: description: Number of active tasks in this job. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -404,6 +464,8 @@ metrics: spark.job.task.result: description: Number of tasks with a specific result in this job. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -413,6 +475,8 @@ metrics: spark.job.stage.active: description: Number of active stages in this job. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -422,6 +486,8 @@ metrics: spark.job.stage.result: description: Number of stages with a specific result in this job. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -432,6 +498,8 @@ metrics: spark.driver.block_manager.disk.usage: description: Disk space used by the BlockManager. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -441,6 +509,8 @@ metrics: spark.driver.block_manager.memory.usage: description: Memory usage for the driver's BlockManager. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -450,6 +520,8 @@ metrics: spark.driver.hive_external_catalog.file_cache_hits: description: Number of file cache hits on the HiveExternalCatalog. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -459,6 +531,8 @@ metrics: spark.driver.hive_external_catalog.files_discovered: description: Number of files discovered while listing the partitions of a table in the Hive metastore enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -468,6 +542,8 @@ metrics: spark.driver.hive_external_catalog.hive_client_calls: description: Number of calls to the underlying Hive Metastore client made by the Spark application. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -477,6 +553,8 @@ metrics: spark.driver.hive_external_catalog.parallel_listing_jobs: description: Number of parallel listing jobs initiated by the HiveExternalCatalog when listing partitions of a table. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -486,6 +564,8 @@ metrics: spark.driver.hive_external_catalog.partitions_fetched: description: Table partitions fetched by the HiveExternalCatalog. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -495,6 +575,8 @@ metrics: spark.driver.code_generator.compilation.count: description: Number of source code compilation operations performed by the CodeGenerator. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -504,6 +586,8 @@ metrics: spark.driver.code_generator.compilation.average_time: description: Average time spent during CodeGenerator source code compilation operations. enabled: true + stability: + level: development gauge: value_type: double unit: ms @@ -511,6 +595,8 @@ metrics: spark.driver.code_generator.generated_class.count: description: Number of classes generated by the CodeGenerator. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -520,6 +606,8 @@ metrics: spark.driver.code_generator.generated_class.average_size: description: Average class size of the classes generated by the CodeGenerator. enabled: true + stability: + level: development gauge: value_type: double unit: bytes @@ -527,6 +615,8 @@ metrics: spark.driver.code_generator.generated_method.count: description: Number of methods generated by the CodeGenerator. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -536,6 +626,8 @@ metrics: spark.driver.code_generator.generated_method.average_size: description: Average method size of the classes generated by the CodeGenerator. enabled: true + stability: + level: development gauge: value_type: double unit: bytes @@ -543,6 +635,8 @@ metrics: spark.driver.code_generator.source_code.operations: description: Number of source code generation operations performed by the CodeGenerator. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -552,6 +646,8 @@ metrics: spark.driver.code_generator.source_code.average_size: description: Average size of the source code generated by a CodeGenerator code generation operation. enabled: true + stability: + level: development gauge: value_type: double unit: bytes @@ -559,6 +655,8 @@ metrics: spark.driver.dag_scheduler.job.active: description: Number of active jobs currently being processed by the DAGScheduler. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -568,6 +666,8 @@ metrics: spark.driver.dag_scheduler.job.count: description: Number of jobs that have been submitted to the DAGScheduler. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -577,6 +677,8 @@ metrics: spark.driver.dag_scheduler.stage.failed: description: Number of failed stages run by the DAGScheduler. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -586,6 +688,8 @@ metrics: spark.driver.dag_scheduler.stage.count: description: Number of stages the DAGScheduler is either running or needs to run. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -595,6 +699,8 @@ metrics: spark.driver.live_listener_bus.posted: description: Number of events that have been posted on the LiveListenerBus. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -604,6 +710,8 @@ metrics: spark.driver.live_listener_bus.processing_time.average: description: Average time taken for the LiveListenerBus to process an event posted to it. enabled: true + stability: + level: development gauge: value_type: double unit: ms @@ -611,6 +719,8 @@ metrics: spark.driver.live_listener_bus.dropped: description: Number of events that have been dropped by the LiveListenerBus. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -620,6 +730,8 @@ metrics: spark.driver.live_listener_bus.queue_size: description: Number of events currently waiting to be processed by the LiveListenerBus. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -629,6 +741,8 @@ metrics: spark.driver.jvm_cpu_time: description: Current CPU time taken by the Spark driver. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -638,6 +752,8 @@ metrics: spark.driver.executor.memory.jvm: description: Amount of memory used by the driver's JVM. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -647,6 +763,8 @@ metrics: spark.driver.executor.memory.execution: description: Amount of execution memory currently used by the driver. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -656,6 +774,8 @@ metrics: spark.driver.executor.memory.storage: description: Amount of storage memory currently used by the driver. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -665,6 +785,8 @@ metrics: spark.driver.executor.memory.pool: description: Amount of pool memory currently used by the driver. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: false @@ -674,6 +796,8 @@ metrics: spark.driver.executor.gc.operations: description: Number of garbage collection operations performed by the driver. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true @@ -683,6 +807,8 @@ metrics: spark.driver.executor.gc.time: description: Total elapsed time during garbage collection operations performed by the driver. enabled: true + stability: + level: development sum: aggregation_temporality: cumulative monotonic: true