elastic · narph · Jan 14, 2020 · Jan 13, 2020 · Jan 13, 2020 · kaiyan-sheng
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -141,6 +141,94 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Metricbeat*
 
+- Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053]
+- Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055]
+- Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055]
+- Add `coredns` metricbeat module. {pull}10585[10585]
+- Add SSL support for Metricbeat HTTP server. {pull}11482[11482] {issue}11457[11457]
+- The `elasticsearch.index` metricset (with `xpack.enabled: true`) now collects `refresh.external_total_time_in_millis` fields from Elasticsearch. {pull}11616[11616]
+- Allow module configurations to have variants {pull}9118[9118]
+- Add `timeseries.instance` field calculation. {pull}10293[10293]
+- Added new disk states and raid level to the system/raid metricset. {pull}11613[11613]
+- Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877]
+- Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878]
+- Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734]
+- Add `regions` in aws module config to specify target regions for querying cloudwatch metrics. {issue}11932[11932] {pull}11956[11956]
+- Keep `etcd` followers members from reporting `leader` metricset events {pull}12004[12004]
+- Add overview dashboard to Consul module {pull}10665[10665]
+- New fields were added in the mysql/status metricset. {pull}12227[12227]
+- Add Kubernetes metricset `proxy`. {pull}12312[12312]
+- Add Kubernetes proxy dashboard to Kubernetes module {pull}12734[12734]
+- Always report Pod UID in the `pod` metricset. {pull}12345[12345]
+- Add Vsphere Virtual Machine operating system to `os` field in Vsphere virtualmachine module. {pull}12391[12391]
+- Add validation for elasticsearch and kibana modules' metricsets when xpack.enabled is set to true. {pull}12386[12386]
+- Add CockroachDB module. {pull}12467[12467]
+- Add support for metricbeat modules based on existing modules (a.k.a. light modules) {issue}12270[12270] {pull}12465[12465]
+- Add a system/entropy metricset {pull}12450[12450]
+- Add kubernetes metricset `controllermanager` {pull}12409[12409]
+- Add Kubernetes controller manager dashboard to Kubernetes module {pull}12744[12744]
+- Allow redis URL format in redis hosts config. {pull}12408[12408]
+- Add tags into ec2 metricset. {issue}[12263]12263 {pull}12372[12372]
+- Add metrics to kubernetes apiserver metricset. {pull}12922[12922]
+- Add kubernetes metricset `scheduler` {pull}12521[12521]
+- Add Kubernetes scheduler dashboard to Kubernetes module {pull}12749[12749]
+- Add `beat` module. {pull}12181[12181] {pull}12615[12615]
+- Collect tags for cloudwatch metricset in aws module. {issue}[12263]12263 {pull}12480[12480]
+- Add AWS RDS metricset. {pull}11620[11620] {issue}10054[10054]
+- Add Oracle Module {pull}11890[11890]
+- Add Oracle Tablespaces Dashboard {pull}12736[12736]
+- Collect client provided name for rabbitmq connection. {issue}12851[12851] {pull}12852[12852]
+- Add support to load default aws config file to get credentials. {pull}12727[12727] {issue}12708[12708]
+- Add statistic option into cloudwatch metricset. {issue}12370[12370] {pull}12840[12840]
+- Add support for kubernetes cronjobs {pull}13001[13001]
+- Add cgroup memory stats to docker/memory metricset {pull}12916[12916]
+- Add AWS elb metricset. {pull}12952[12952] {issue}11701[11701]
+- Add AWS ebs metricset. {pull}13167[13167] {issue}11699[11699]
+- Add `metricset.period` field with the configured fetching period. {pull}13242[13242] {issue}12616[12616]
+- Add rate metrics for ec2 metricset. {pull}13203[13203]
+- Add refresh list of perf counters at every fetch {issue}13091[13091]
+- Add Performance metricset to Oracle module {pull}12547[12547]
+- Add proc/vmstat data to the system/memory metricset on linux {pull}13322[13322]
+- Use DefaultMetaGeneratorConfig in MetadataEnrichers to initialize configurations {pull}13414[13414]
+- Add module for statsd. {pull}13109[13109]
+- Add support for NATS version 2. {pull}13601[13601]
+- Add `docker.cpu.*.norm.pct` metrics for `cpu` metricset of Docker Metricbeat module. {pull}13695[13695]
+- Add `instance` label by default when using Prometheus collector. {pull}13737[13737]
+- Add azure module. {pull}13196[13196] {pull}13859[13859] {pull}13988[13988]
+- Add Apache Tomcat module {pull}13491[13491]
+- Add ECS `container.id` and `container.runtime` to kubernetes `state_container` metricset. {pull}13884[13884]
+- Add `job` label by default when using Prometheus collector. {pull}13878[13878]
+- Add `state_resourcequota` metricset for Kubernetes module. {pull}13693[13693]
+- Add tags filter in ec2 metricset. {pull}13872[13872] {issue}13145[13145]
+- Add cloud.account.id and cloud.account.name into events from aws module. {issue}13551[13551] {pull}13558[13558]
+- Add `metrics_path` as known hint for autodiscovery {pull}13996[13996]
+- Leverage KUBECONFIG when creating k8s client. {pull}13916[13916]
+- Add ability to filter by tags for cloudwatch metricset. {pull}13758[13758] {issue}13145[13145]
+- Release cloudwatch, s3_daily_storage, s3_request, sqs and rds metricset as GA. {pull}14114[14114] {issue}14059[14059]
+- Add Oracle overview dashboard {pull}14021[14021]
+- Release CoreDNS module as GA. {pull}14308[14308]
+- Release CouchDB module as GA. {pull}14300[14300]
+- Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
+- Add support for Application ELB and Network ELB. {pull}14123[14123] {issue}13538[13538] {issue}13539[13539]
+- Release aws ebs metricset as GA. {pull}14312[14312] {issue}14060[14060]
+- Add `connection.state` field for RabbitMQ module. {pull}13981[13981]
+- Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
+- Add Kafka JMX metricsets. {pull}14330[14330]
+- Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642]
+- Release kubernetes modules `controllermanager`, `scheduler`, `proxy`, `state_cronjob` and `state_resourcequota` as GA. {pull}14584[14584]
+- Add module for ActiveMQ. {pull}14580[14580]
+- Enable script processor. {pull}14711[14711]
+- Enable wildcard for cloudwatch metricset namespace. {pull}14971[14971] {issue}14965[14965]
+- Add `kube-state-metrics` `state_service` metrics for kubernetes module. {pull}14794[14794]
+- Add `kube-state-metrics` `state_persistentvolume` metrics for kubernetes module. {pull}14859[14859]
+- Add `kube-state-metrics` `state_persistentvolumeclaim` metrics for kubernetes module. {pull}15066[15066]
+- Add usage metricset in aws modules. {pull}14925[14925] {issue}14935[14935]
+- Add billing metricset in aws modules. {pull}14801[14801] {issue}14934[14934]
+- Add AWS SNS metricset. {pull}14946[14946]
+- Add overview dashboard for AWS SNS module {pull}14977[14977]
+- Add `index` option to all modules to specify a module-specific output index. {pull}15100[15100]
+- Add a `system/service` metricset for systemd data. {pull}14206[14206]
+- Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342]
 - Add cost warnings for the azure module. {pull}15356[15356]
 - Release elb module as GA. {pull}15485[15485]
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -2130,84 +2130,84 @@ azure module
 
 
 
-*`azure.namespace`*::
-+
---
-The namespace selected
+[float]
+=== resource
 
+The resource specified
 
-type: keyword
 
---
 
-*`azure.subscription_id`*::
+*`azure.resource.name`*::
 +
 --
-The subscription ID
+The name of the resource
 
 
 type: keyword
 
 --
 
-*`azure.dimensions.*`*::
+*`azure.resource.type`*::
 +
 --
-Azure metric dimensions.
+The type of the resource
 
 
-type: object
+type: keyword
 
 --
 
-[float]
-=== resource
+*`azure.resource.group`*::
++
+--
+The resource group
 
-The resource specified
 
+type: keyword
 
+--
 
-*`azure.resource.name`*::
+*`azure.resource.tags.*`*::
 +
 --
-The name of the resource
+Azure resource tags.
 
 
-type: keyword
+type: object
 
 --
 
-*`azure.resource.type`*::
+*`azure.namespace`*::
 +
 --
-The type of the resource
+The namespace selected
 
 
 type: keyword
 
 --
 
-*`azure.resource.group`*::
+*`azure.subscription_id`*::
 +
 --
-The resource group
+The subscription ID
 
 
 type: keyword
 
 --
 
-*`azure.resource.tags.*`*::
+*`azure.dimensions.*`*::
 +
 --
-Azure resource tags.
+Azure metric dimensions.
 
 
 type: object
 
 --
 
-*`azure.resource.compute_vm.*.*`*::
+*`azure.compute_vm.*.*`*::
 +
 --
 compute_vm
@@ -2217,7 +2217,7 @@ type: object
 
 --
 
-*`azure.resource.compute_vm_scaleset.*.*`*::
+*`azure.compute_vm_scaleset.*.*`*::
 +
 --
 compute_vm_scaleset
@@ -2234,12 +2234,22 @@ monitor
 
 
 
-*`azure.resource.monitor.metrics.*.*`*::
+*`azure.monitor.metrics.*.*`*::
 +
 --
 Metrics returned.
 
 
+type: object
+
+--
+
+*`azure.storage.*.*`*::
++
+--
+storage account
+
+
 type: object
 
 --

diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc
@@ -78,6 +78,11 @@ so the `period` for `compute_vm` metricset  should be `300s` or multiples of `30
 This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes,
 so the `period` for `compute_vm_scaleset` metricset  should be `300s` or multiples of `300s`.
 
+[float]
+=== `storage`
+This metricset will collect metrics from the storage accounts, these metrics will have a timegrain every 5 minutes,
+so the `period` for `storage` metricset  should be `300s` or multiples of `300s`.
+
 
 [float]
 == Additional notes about metrics and costs
@@ -130,6 +135,16 @@ metricbeat.modules:
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - storage
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 ----
 
 [float]
@@ -143,9 +158,13 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-azure-monitor,monitor>>
 
+* <<metricbeat-metricset-azure-storage,storage>>
+
 include::azure/compute_vm.asciidoc[]
 
 include::azure/compute_vm_scaleset.asciidoc[]
 
 include::azure/monitor.asciidoc[]
 
+include::azure/storage.asciidoc[]
+
diff --git a/metricbeat/docs/modules/azure/storage.asciidoc b/metricbeat/docs/modules/azure/storage.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-azure-storage]]
+=== azure storage metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-azure,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/azure/storage/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -28,9 +28,10 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-aws-sqs,sqs>>   
 |<<metricbeat-metricset-aws-usage,usage>> beta[]  
 |<<metricbeat-module-azure,azure>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.3+| .3+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
+.4+| .4+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
 |<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>> beta[]  
 |<<metricbeat-metricset-azure-monitor,monitor>> beta[]  
+|<<metricbeat-metricset-azure-storage,storage>> beta[]  
 |<<metricbeat-module-beat,Beat>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .2+| .2+|  |<<metricbeat-metricset-beat-state,state>>   
 |<<metricbeat-metricset-beat-stats,stats>>   

diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -281,6 +281,16 @@ metricbeat.modules:
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 
+- module: azure
+  metricsets:
+  - storage
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
 #--------------------------------- Beat Module ---------------------------------
 - module: beat
   metricsets:

diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml
@@ -32,3 +32,13 @@
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - storage
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml
@@ -35,3 +35,14 @@
 #  tenant_id: '${AZURE_TENANT_ID:""}'
 #  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 #  refresh_list_interval: 600s
+
+#- module: azure
+#  metricsets:
+#  - storage
+#  enabled: true
+#  period: 300s
+#  client_id: '${AZURE_CLIENT_ID:""}'
+#  client_secret: '${AZURE_CLIENT_SECRET:""}'
+#  tenant_id: '${AZURE_TENANT_ID:""}'
+#  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+#  refresh_list_interval: 600s
diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc
@@ -68,6 +68,11 @@ so the `period` for `compute_vm` metricset  should be `300s` or multiples of `30
 This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes,
 so the `period` for `compute_vm_scaleset` metricset  should be `300s` or multiples of `300s`.
 
+[float]
+=== `storage`
+This metricset will collect metrics from the storage accounts, these metrics will have a timegrain every 5 minutes,
+so the `period` for `storage` metricset  should be `300s` or multiples of `300s`.
+
 
 [float]
 == Additional notes about metrics and costs