From 13d4b99bdf8357a64729ccc6ffb489c2b4b02bc4 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 11:20:08 +0100 Subject: [PATCH 1/7] work on GA --- ...etricbeat-azure-blob-storage-overview.json | 10 +- ...etricbeat-azure-file-storage-overview.json | 18 +- ...tricbeat-azure-queue-storage-overview.json | 10 +- .../Metricbeat-azure-storage-overview.json | 14 +- ...tricbeat-azure-table-storage-overview.json | 10 +- ...icbeat-azure-vm-guestmetrics-overview.json | 2 +- .../Metricbeat-azure-vm-overview.json | 18 +- .../Metricbeat-azure-vmss-overview.json | 16 +- x-pack/metricbeat/module/azure/azure.go | 10 +- x-pack/metricbeat/module/azure/client.go | 32 +-- x-pack/metricbeat/module/azure/client_test.go | 2 +- .../metricbeat/module/azure/client_utils.go | 20 +- .../module/azure/client_utils_test.go | 12 +- .../module/azure/compute_vm/_meta/data.json | 217 +++++++++--------- .../module/azure/compute_vm/client_helper.go | 28 ++- .../azure/compute_vm/client_helper_test.go | 10 +- .../compute_vm/compute_vm_integration_test.go | 39 ++++ .../azure/compute_vm_scaleset/_meta/data.json | 176 +++++++------- .../compute_vm_scaleset/client_helper.go | 12 +- .../compute_vm_scaleset/client_helper_test.go | 10 +- .../compute_vm_scaleset_integration_test.go | 39 ++++ x-pack/metricbeat/module/azure/data.go | 13 +- .../database_account_integration_test.go | 54 ++--- .../metricbeat/module/azure/mock_service.go | 22 +- .../module/azure/monitor/_meta/data.json | 84 +++---- .../module/azure/monitor/client_helper.go | 8 +- .../azure/monitor/client_helper_test.go | 4 +- .../azure/monitor/monitor_integration_test.go | 43 ++++ .../module/azure/monitor_service.go | 43 ++-- x-pack/metricbeat/module/azure/resources.go | 7 +- .../module/azure/service_interface.go | 9 +- .../module/azure/storage/_meta/data.json | 115 +++------- .../module/azure/storage/client_helper.go | 2 +- .../azure/storage/client_helper_test.go | 4 +- .../azure/storage/storage_integration_test.go | 39 ++++ .../module/azure/test/integration.go | 36 +++ 36 files changed, 687 insertions(+), 501 deletions(-) create mode 100644 x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/storage/storage_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/test/integration.go diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json index b29fd135f815..aad73bdc9224 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json @@ -483,7 +483,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -513,7 +513,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -616,7 +616,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -767,7 +767,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1048,7 +1048,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json index a2c911a036c0..b0ea29b3c66c 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json @@ -523,7 +523,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -553,7 +553,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -656,7 +656,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -759,7 +759,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -910,7 +910,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1165,7 +1165,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1195,7 +1195,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -1272,7 +1272,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1302,7 +1302,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json index e447d41aa23d..8783bbaa8a60 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json @@ -483,7 +483,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -513,7 +513,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -616,7 +616,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -719,7 +719,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -870,7 +870,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json index 4dcabeb10e1f..4de70977eeb6 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json @@ -474,7 +474,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -504,7 +504,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -607,7 +607,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -710,7 +710,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -861,7 +861,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1116,7 +1116,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1146,7 +1146,7 @@ "split_color_mode": "gradient", "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json index 6b2fcd65dc0b..ad7bbfe89705 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json @@ -483,7 +483,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -513,7 +513,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -616,7 +616,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -719,7 +719,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -870,7 +870,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json index 873616c609ca..0613a7e4a8ea 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json @@ -251,7 +251,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1570774891724", "indexPatternRefName": "control_2_index_pattern", "label": "VM Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json index e73bc6a1e5e1..08b5b0446332 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json @@ -266,7 +266,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1570774891724", "indexPatternRefName": "control_2_index_pattern", "label": "VM Name", @@ -372,7 +372,7 @@ "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}%" } ], @@ -447,7 +447,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -522,7 +522,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -598,7 +598,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -673,7 +673,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -749,7 +749,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -825,7 +825,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -901,7 +901,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json index fe615da131fa..7e8ed312b79c 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json @@ -374,7 +374,7 @@ "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -449,7 +449,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -524,7 +524,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -600,7 +600,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -676,7 +676,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -752,7 +752,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -828,7 +828,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -904,7 +904,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 1d08b60951f1..1c01c203ae98 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -17,10 +17,10 @@ import ( // Config options type Config struct { - ClientID string `config:"client_id" validate:"required"` + ClientId string `config:"client_id" validate:"required"` ClientSecret string `config:"client_secret" validate:"required"` - TenantID string `config:"tenant_id" validate:"required"` - SubscriptionID string `config:"subscription_id" validate:"required"` + TenantId string `config:"tenant_id" validate:"required"` + SubscriptionId string `config:"subscription_id" validate:"required"` Period time.Duration `config:"period" validate:"nonzero,required"` Resources []ResourceConfig `config:"resources"` RefreshListInterval time.Duration `config:"refresh_list_interval"` @@ -29,7 +29,7 @@ type Config struct { // ResourceConfig contains resource and metric list specific configuration. type ResourceConfig struct { - ID []string `config:"resource_id"` + Id []string `config:"resource_id"` Group []string `config:"resource_group"` Metrics []MetricConfig `config:"metrics"` Type string `config:"resource_type"` @@ -106,7 +106,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription var resources []ResourceConfig for _, resource := range config.Resources { - if len(resource.Group) != 0 || len(resource.ID) != 0 { + if len(resource.Group) != 0 || len(resource.Id) != 0 { resources = append(resources, resource) } } diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 5a3871f1046b..a94b10d07ddb 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -31,7 +31,7 @@ type mapResourceMetrics func(client *Client, resources []resources.GenericResour // NewClient instantiates the an Azure monitoring client func NewClient(config Config) (*Client, error) { - azureMonitorService, err := NewService(config.ClientID, config.ClientSecret, config.TenantID, config.SubscriptionID) + azureMonitorService, err := NewService(config.ClientId, config.ClientSecret, config.TenantId, config.SubscriptionId) if err != nil { return nil, err } @@ -57,14 +57,14 @@ func (client *Client) InitResources(fn mapResourceMetrics, report mb.ReporterV2) var metrics []Metric for _, resource := range client.Config.Resources { // retrieve azure resources information - resourceList, err := client.AzureMonitorService.GetResourceDefinitions(resource.ID, resource.Group, resource.Type, resource.Query) + resourceList, err := client.AzureMonitorService.GetResourceDefinitions(resource.Id, resource.Group, resource.Type, resource.Query) if err != nil { err = errors.Wrap(err, "failed to retrieve resources") return err } if len(resourceList.Values()) == 0 { err = errors.Errorf("failed to retrieve resources: No resources returned using the configuration options resource ID %s, resource group %s, resource type %s, resource query %s", - resource.ID, resource.Group, resource.Type, resource.Query) + resource.Id, resource.Group, resource.Type, resource.Query) client.Log.Error(err) continue } @@ -107,20 +107,20 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } filter = strings.Join(filterList, " AND ") } - resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names, + resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubId, metric.Namespace, metric.TimeGrain, timespan, metric.Names, metric.Aggregations, filter) if err != nil { - err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubID, metric.Namespace) + err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubId, metric.Namespace) client.Log.Error(err) report.Error(err) } else { - if metric.TimeGrain == "" { - metric.TimeGrain = timegrain - } for i, currentMetric := range client.Resources.Metrics { if matchMetrics(currentMetric, metric) { current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) client.Resources.Metrics[i].Values = current + if client.Resources.Metrics[i].TimeGrain == "" { + client.Resources.Metrics[i].TimeGrain = timegrain + } resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i]) } } @@ -130,17 +130,19 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } // CreateMetric function will create a client metric based on the resource and metrics configured -func (client *Client) CreateMetric(selectedResourceID string, resource resources.GenericResource, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { +func (client *Client) CreateMetric(selectedResourceID string, resource resources.GenericResource, resourceSize string, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { met := Metric{ Resource: Resource{ - SubID: selectedResourceID, - ID: *resource.ID, + SubId: selectedResourceID, + Id: *resource.ID, Name: *resource.Name, Location: *resource.Location, Type: *resource.Type, - Group: getResourceGroupFromID(*resource.ID), + Group: getResourceGroupFromId(*resource.ID), Tags: mapTags(resource.Tags), - Subscription: client.Config.SubscriptionID}, + Subscription: client.Config.SubscriptionId, + Size: resourceSize, + }, Namespace: namespace, Names: metrics, Dimensions: dimensions, @@ -156,7 +158,7 @@ func (client *Client) CreateMetric(selectedResourceID string, resource resources } // MapMetricByPrimaryAggregation will map the primary aggregation of the metric definition to the client metric -func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefinition, resource resources.GenericResource, selectedResourceID string, namespace string, dim []Dimension, timegrain string) []Metric { +func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDefinition, resource resources.GenericResource, selectedResourceID string, resourceSize string, namespace string, dim []Dimension, timegrain string) []Metric { var clientMetrics []Metric metricGroups := make(map[string][]insights.MetricDefinition) @@ -171,7 +173,7 @@ func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefi if selectedResourceID == "" { selectedResourceID = *resource.ID } - clientMetrics = append(clientMetrics, client.CreateMetric(selectedResourceID, resource, namespace, metricNames, key, dim, timegrain)) + clientMetrics = append(clientMetrics, client.CreateMetric(selectedResourceID, resource, resourceSize, namespace, metricNames, key, dim, timegrain)) } return clientMetrics } diff --git a/x-pack/metricbeat/module/azure/client_test.go b/x-pack/metricbeat/module/azure/client_test.go index 559b4efa7eb7..970a3d6e7946 100644 --- a/x-pack/metricbeat/module/azure/client_test.go +++ b/x-pack/metricbeat/module/azure/client_test.go @@ -17,7 +17,7 @@ import ( var ( resourceIDConfig = Config{ Resources: []ResourceConfig{ - {ID: []string{"123"}, + {Id: []string{"123"}, Metrics: []MetricConfig{ { Name: []string{"hello", "test"}, diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index b03964044ea2..90f7310dc1fe 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -87,15 +87,15 @@ func metricIsEmpty(metric insights.MetricValue) bool { // matchMetrics will compare current metrics func matchMetrics(prevMet Metric, met Metric) bool { - if prevMet.Namespace == met.Namespace && reflect.DeepEqual(prevMet.Names, met.Names) && prevMet.Resource.ID == met.Resource.ID && + if prevMet.Namespace == met.Namespace && reflect.DeepEqual(prevMet.Names, met.Names) && prevMet.Resource.Id == met.Resource.Id && prevMet.Aggregations == met.Aggregations && prevMet.TimeGrain == met.TimeGrain { return true } return false } -// getResourceGroupFormID maps resource group from resource ID -func getResourceGroupFromID(path string) string { +// getResourceGroupFromId maps resource group from resource ID +func getResourceGroupFromId(path string) string { params := strings.Split(path, "/") for i, param := range params { if param == "resourceGroups" { @@ -105,8 +105,8 @@ func getResourceGroupFromID(path string) string { return "" } -// getResourceNameFormID maps resource group from resource ID -func getResourceTypeFromID(path string) string { +// getResourceTypeFromId maps resource group from resource ID +func getResourceTypeFromId(path string) string { params := strings.Split(path, "/") for i, param := range params { if param == "providers" { @@ -116,8 +116,8 @@ func getResourceTypeFromID(path string) string { return "" } -// getResourceNameFormID maps resource group from resource ID -func getResourceNameFromID(path string) string { +// getResourceNameFormId maps resource group from resource ID +func getResourceNameFromId(path string) string { params := strings.Split(path, "/") if strings.HasSuffix(path, "/") { return params[len(params)-2] @@ -181,10 +181,10 @@ func convertTimegrainToDuration(timegrain string) time.Duration { func groupMetricsByResource(metrics []Metric) map[string][]Metric { grouped := make(map[string][]Metric) for _, metric := range metrics { - if _, ok := grouped[metric.Resource.ID]; !ok { - grouped[metric.Resource.ID] = make([]Metric, 0) + if _, ok := grouped[metric.Resource.Id]; !ok { + grouped[metric.Resource.Id] = make([]Metric, 0) } - grouped[metric.Resource.ID] = append(grouped[metric.Resource.ID], metric) + grouped[metric.Resource.Id] = append(grouped[metric.Resource.Id], metric) } return grouped } diff --git a/x-pack/metricbeat/module/azure/client_utils_test.go b/x-pack/metricbeat/module/azure/client_utils_test.go index a63034ff77d7..0a2ee2cb54a6 100644 --- a/x-pack/metricbeat/module/azure/client_utils_test.go +++ b/x-pack/metricbeat/module/azure/client_utils_test.go @@ -60,7 +60,7 @@ func TestMetricExists(t *testing.T) { func TestMatchMetrics(t *testing.T) { prev := Metric{ - Resource: Resource{Name: "vm", Group: "group", ID: "id"}, + Resource: Resource{Name: "vm", Group: "group", Id: "id"}, Namespace: "namespace", Names: []string{"TotalRequests,Capacity"}, Aggregations: "Average,Total", @@ -69,7 +69,7 @@ func TestMatchMetrics(t *testing.T) { TimeGrain: "1PM", } current := Metric{ - Resource: Resource{Name: "vm", Group: "group", ID: "id"}, + Resource: Resource{Name: "vm", Group: "group", Id: "id"}, Namespace: "namespace", Names: []string{"TotalRequests,Capacity"}, Aggregations: "Average,Total", @@ -79,7 +79,7 @@ func TestMatchMetrics(t *testing.T) { } result := matchMetrics(prev, current) assert.True(t, result) - current.Resource.ID = "id1" + current.Resource.Id = "id1" result = matchMetrics(prev, current) assert.False(t, result) } @@ -107,19 +107,19 @@ func TestMetricIsEmpty(t *testing.T) { func TestGetResourceGroupFromID(t *testing.T) { path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - group := getResourceGroupFromID(path) + group := getResourceGroupFromId(path) assert.Equal(t, group, "obs-infrastructure") } func TestGetResourceTypeFromID(t *testing.T) { path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - rType := getResourceTypeFromID(path) + rType := getResourceTypeFromId(path) assert.Equal(t, rType, "Microsoft.Compute/virtualMachines") } func TestGetResourceNameFromID(t *testing.T) { path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - name := getResourceNameFromID(path) + name := getResourceNameFromId(path) assert.Equal(t, name, "obstestmemleak") } diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index f0692fae3b61..4a6319ef319d 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -1,154 +1,145 @@ { - "@timestamp":"2019-09-20T11:02:00.000Z", - "azure":{ - "timegrain" : "PT5M", - "resource":{ - "name":"obslinux", - "type":"Microsoft.Compute/virtualMachines", - "group":"resource-group" - }, - "subscription_id":"123456-qwer-1234-5678-12345678", - "namespace":"Microsoft.Compute/virtualMachines", - "compute_vm":{ - "inbound_flows_maximum_creation_rate":{ - "avg":14.6 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "compute_vm": { + "data_disk_queue_depth": { + "avg": 0 + }, + "data_disk_read_bytes_per_sec": { + "avg": 0 + }, + "data_disk_read_operations_per_sec": { + "avg": 0 }, - "os_per_disk_write_bytes_per_sec":{ - "avg":15261.86 + "data_disk_write_bytes_per_sec": { + "avg": 0 }, - "outbound_flows_maximum_creation_rate":{ - "avg":18.6 + "data_disk_write_operations_per_sec": { + "avg": 0 }, - "os_per_disk_read_operations_per_sec":{ - "avg":0 + "disk_read_bytes": { + "total": 32768.02 }, - "data_disk_queue_depth":{ - "avg":0 + "disk_read_operations_per_sec": { + "avg": 0.008 }, - "per_disk_read_operations_per_sec":{ - "avg":0 + "disk_write_bytes": { + "total": 19635551.44 }, - "inbound_flows":{ - "avg":60.4 + "disk_write_operations_per_sec": { + "avg": 2.8600000000000003 }, - "os_disk_write_operations_per_sec":{ - "avg":1.0175 + "inbound_flows": { + "avg": 52.2 }, - "per_disk_read_bytes_per_sec":{ - "avg":0 + "inbound_flows_maximum_creation_rate": { + "avg": 11.6 }, - "disk_write_operations_per_sec":{ - "avg":1.03 + "network_in": { + "total": 161463 }, - "outbound_flows":{ - "avg":60.4 + "network_in_total": { + "total": 17721837 }, - "network_in":{ - "total":3624009.0 + "network_out": { + "total": 129547 }, - "os_per_disk_qd":{ - "avg":0 + "network_out_total": { + "total": 1740212 }, - "data_disk_read_bytes_per_sec":{ - "avg":0 + "os_disk_queue_depth": { + "avg": 0.01 }, - "disk_read_operations_per_sec":{ - "avg":0 + "os_disk_read_bytes_per_sec": { + "avg": 109.226 }, - "os_disk_read_operations_per_sec":{ - "avg":0 + "os_disk_read_operations_per_sec": { + "avg": 0.008 }, - "per_disk_qd":{ - "avg":0 + "os_disk_write_bytes_per_sec": { + "avg": 65451.831999999995 }, - "os_disk_write_bytes_per_sec":{ - "avg":15261.86 + "os_disk_write_operations_per_sec": { + "avg": 2.8600000000000003 }, - "disk_write_bytes":{ - "total":2806782.57 + "os_per_disk_qd": { + "avg": 0.01 }, - "per_disk_write_operations_per_sec":{ - "avg":0 + "os_per_disk_read_bytes_per_sec": { + "avg": 109.226 }, - "network_out":{ - "total":704862 + "os_per_disk_read_operations_per_sec": { + "avg": 0.008 }, - "os_disk_read_bytes_per_sec":{ - "avg":0 + "os_per_disk_write_bytes_per_sec": { + "avg": 65451.831999999995 }, - "os_per_disk_read_bytes_per_sec":{ - "avg":0 + "os_per_disk_write_operations_per_sec": { + "avg": 2.8600000000000003 }, - "os_per_disk_write_operations_per_sec":{ - "avg":1.0175 + "outbound_flows": { + "avg": 52.2 }, - "percentage_cpu":{ - "avg":7.683333333333334 + "outbound_flows_maximum_creation_rate": { + "avg": 11.6 }, - "data_disk_write_operations_per_sec":{ - "avg":0 + "per_disk_qd": { + "avg": 0 }, - "network_in_total":{ - "total":6787492.0 + "per_disk_read_bytes_per_sec": { + "avg": 0 }, - "disk_read_bytes":{ - "total":0 + "per_disk_read_operations_per_sec": { + "avg": 0 }, - "data_disk_write_bytes_per_sec":{ - "avg":0 + "per_disk_write_bytes_per_sec": { + "avg": 0 }, - "per_disk_write_bytes_per_sec":{ - "avg":0 + "per_disk_write_operations_per_sec": { + "avg": 0 }, - "data_disk_read_operations_per_sec":{ - "avg":0 + "percentage_cpu": { + "avg": 2.489 }, - "os_disk_queue_depth":{ - "avg":0 + "premium_os_disk_cache_read_hit": { + "avg": 100 }, - "network_out_total":{ - "total":1889767.0 + "premium_os_disk_cache_read_miss": { + "avg": 0 } - } - }, - "service":{ - "type":"azure" - }, - "cloud":{ - "provider":"azure", - "region":"westeurope" - }, - "ecs":{ - "version":"1.1.0" + }, + "namespace": "Microsoft.Compute/virtualMachines", + "resource": { + "group": "obs-infrastructure", + "tags": { + "provider": "648D2193-0CE0-4EFB-8A82-AF9792184FD9" + }, + "type": "Microsoft.Compute/virtualMachines" + }, + "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "timegrain": "PT5M" }, - "host":{ - "architecture":"x86_64", - "os":{ - "build":"17134.1006", - "platform":"windows", - "version":"10.0", - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + "cloud": { + "instance": { + "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/elkdata-0", + "name": "elkdata-0" + }, + "machine": { + "type": "Standard_DS1_v2" }, - "id":"123456-qwer-1234-5678-12345678", - "name":"DESKTOP-RFOOE09", - "hostname":"DESKTOP-RFOOE09" + "provider": "azure", + "region": "eastus" }, - "agent":{ - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat", - "ephemeral_id":"123456-qwer-1234-5678-12345678" + "event": { + "dataset": "azure.compute_vm", + "duration": 115000, + "module": "azure" }, - "event":{ - "duration":4611928900, - "dataset":"azure.compute_vm", - "module":"azure" + "metricset": { + "name": "compute_vm", + "period": 10000 }, - "metricset":{ - "name":"compute_vm", - "period":300000 + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go index d4b4e241af45..563a61ab34b5 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go @@ -19,6 +19,8 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res return nil, nil } for _, resource := range resources { + // return resource size + resourceSize := mapResourceSize(resource, client) // return all namespaces supported for this resource namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) if err != nil { @@ -38,8 +40,32 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) } // map azure metric definitions to client metrics - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, filteredMetricDefinitions, resource, "", *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(filteredMetricDefinitions, resource, "", resourceSize, *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) } } return metrics, nil } + +// mapResourceSize func will try to map if existing the resource size +func mapResourceSize(resource resources.GenericResource, client *azure.Client) string { + if resource.Sku != nil && resource.Sku.Name != nil { + return *resource.Sku.Name + } + if resource.Sku == nil && resource.Properties == nil { + expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(*resource.ID) + if err != nil { + client.Log.Error(err, "could not retrieve the resource details by resource ID %s", *resource.ID) + return "" + } + if expandedResource.Properties != nil { + if properties, ok := expandedResource.Properties.(map[string]interface{}); ok { + if hardware, ok := properties["hardwareProfile"]; ok { + if vmSize, ok := hardware.(map[string]interface{})["vmSize"]; ok { + return vmSize.(string) + } + } + } + } + } + return "" +} diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go index 1d24f6b65aaf..16957b0bf73d 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go @@ -81,8 +81,11 @@ func TestMapMetric(t *testing.T) { metricConfig := azure.MetricConfig{Name: []string{"*"}} var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} client := azure.NewMockClient() + res := resource + res.Properties = map[string]interface{}{"hardwareProfile": map[string]interface{}{"vmSize": "A1Basic"}} t.Run("return error when the metric namespaces api call returns an error", func(t *testing.T) { m := &azure.MockService{} + m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) m.On("GetMetricNamespaces", mock.Anything).Return(insights.MetricNamespaceCollection{}, errors.New("invalid resource ID")) client.AzureMonitorService = m metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) @@ -93,6 +96,7 @@ func TestMapMetric(t *testing.T) { }) t.Run("return error when no metric definitions were found", func(t *testing.T) { m := &azure.MockService{} + m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) client.AzureMonitorService = m @@ -104,20 +108,22 @@ func TestMapMetric(t *testing.T) { }) t.Run("return mapped metrics correctly", func(t *testing.T) { m := &azure.MockService{} + m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Id, "123") assert.Equal(t, metrics[1].Resource.Name, "resourceName") assert.Equal(t, metrics[1].Resource.Type, "resourceType") assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].Resource.Size, "A1Basic") assert.Equal(t, metrics[1].Namespace, "namespace") assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go new file mode 100644 index 000000000000..107becd51613 --- /dev/null +++ b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package compute_vm + +import ( + "testing" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("compute_vm") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("compute_vm") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index ac7687bd5394..405c0c740e44 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -1,121 +1,113 @@ { - "@timestamp" : "2019-09-20T11:13:00.000Z", - "azure" : { - "timegrain" : "PT5M", - "resource" : { - "name" : "winvmss", - "type" : "Microsoft.Compute/virtualMachineScaleSets", - "group" : "resource-group" - }, - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.Compute/virtualMachineScaleSets", - "dimensions" : { - "vmname" : "winvmss_3" - }, - "compute_vm_scaleset" : { - "outbound_flows" : { - "avg" : 10.6 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "compute_vm_scaleset": { + "data_disk_queue_depth": { + "avg": 0 + }, + "data_disk_read_bytes_per_sec": { + "avg": 0 }, - "disk_write_bytes" : { - "total" : 2.838777151E7 + "data_disk_read_operations_per_sec": { + "avg": 0 }, - "disk_read_operations_per_sec" : { - "avg" : 3.53 + "data_disk_write_bytes_per_sec": { + "avg": 0 }, - "outbound_flows_maximum_creation_rate" : { - "avg" : 0.2 + "data_disk_write_operations_per_sec": { + "avg": 0 }, - "data_disk_read_bytes_per_sec" : { - "avg" : 5051.38125 + "disk_read_bytes": { + "total": 0 }, - "os_disk_read_bytes_per_sec" : { - "avg" : 122285.5025 + "disk_read_operations_per_sec": { + "avg": 0 }, - "data_disk_write_bytes_per_sec" : { - "avg" : 0 + "disk_write_bytes": { + "total": 1180450.5 }, - "data_disk_read_operations_per_sec" : { - "avg" : 0.495 + "disk_write_operations_per_sec": { + "avg": 0.6839999999999999 }, - "disk_read_bytes" : { - "total" : 4.081481671E7 + "inbound_flows": { + "avg": 36.8 }, - "disk_write_operations_per_sec" : { - "avg" : 3.132 + "inbound_flows_maximum_creation_rate": { + "avg": 6.6 }, - "network_out_total" : { - "total" : 1509069.0 + "network_in": { + "total": 1158350 }, - "inbound_flows" : { - "avg" : 10.6 + "network_in_total": { + "total": 1737301 }, - "os_disk_read_operations_per_sec" : { - "avg" : 1.805 + "network_out": { + "total": 175875 }, - "data_disk_queue_depth" : { - "avg" : 0 + "network_out_total": { + "total": 646949 }, - "inbound_flows_maximum_creation_rate" : { - "avg" : 0.2 + "os_disk_queue_depth": { + "avg": 0 }, - "data_disk_write_operations_per_sec" : { - "avg" : 0 + "os_disk_read_bytes_per_sec": { + "avg": 0 }, - "os_disk_write_bytes_per_sec" : { - "avg" : 95619.85625 + "os_disk_read_operations_per_sec": { + "avg": 0 }, - "percentage_cpu" : { - "avg" : 1.592 + "os_disk_write_bytes_per_sec": { + "avg": 3934.8300000000004 }, - "os_disk_write_operations_per_sec" : { - "avg" : 3.045 + "os_disk_write_operations_per_sec": { + "avg": 0.6839999999999999 }, - "network_in_total" : { - "total" : 794397 + "outbound_flows": { + "avg": 37 }, - "os_disk_queue_depth" : { - "avg" : 0.005 + "outbound_flows_maximum_creation_rate": { + "avg": 7.6 + }, + "percentage_cpu": { + "avg": 0.30125 } - } - }, - "service" : { - "type" : "azure" - }, - "ecs" : { - "version" : "1.1.0" - }, - "host" : { - "architecture" : "x86_64", - "os" : { - "build" : "17134.1006", - "platform" : "windows", - "version" : "10.0", - "family" : "windows", - "name" : "Windows 10 Pro", - "kernel" : "10.0.17134.1006 (WinBuild.160101.0800)" }, - "id" : "123456-qwer-1234-5678-12345678", - "hostname" : "DESKTOP-RFOOE09", - "name" : "DESKTOP-RFOOE09" + "dimensions": { + "vmname": "ecescaletest_3" + }, + "namespace": "Microsoft.Compute/virtualMachineScaleSets", + "resource": { + "group": "obs-infrastructure", + "tags": { + "environment": "staging", + "role": "allocator" + }, + "type": "Microsoft.Compute/virtualMachineScaleSets" + }, + "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "timegrain": "PT5M" }, - "agent" : { - "version" : "8.0.0", - "type" : "metricbeat", - "ephemeral_id" : "123456-qwer-1234-5678-12345678", - "hostname" : "DESKTOP-RFOOE09", - "id" : "123456-qwer-1234-5678-12345678" + "cloud": { + "instance": { + "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/ecescaletest", + "name": "ecescaletest" + }, + "machine": { + "type": "Standard_D4s_v3" + }, + "provider": "azure", + "region": "eastus2" }, - "cloud" : { - "provider" : "azure", - "region" : "westeurope" + "event": { + "dataset": "azure.compute_vm_scaleset", + "duration": 115000, + "module": "azure" }, - "event" : { - "dataset" : "azure.compute_vm_scaleset", - "module" : "azure", - "duration" : 69629063400 + "metricset": { + "name": "compute_vm_scaleset", + "period": 10000 }, - "metricset" : { - "name" : "compute_vm_scaleset", - "period" : 300000 + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index aa5e699109f9..0021ebd552ea 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -25,6 +25,8 @@ const ( func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric for _, resource := range resources { + // return resource size + resourceSize := mapResourceSize(resource) for _, metric := range resourceConfig.Metrics { metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) if err != nil { @@ -76,7 +78,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if key != azure.NoDimension { dimensions = []azure.Dimension{{Name: key, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, defaultTimeGrain)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(metricGroup, resource, "", resourceSize, metric.Namespace, dimensions, defaultTimeGrain)...) } } } @@ -92,3 +94,11 @@ func containsDimension(dimension string, dimensions []insights.LocalizableString } return false } + +// mapResourceSize func will try to map if existing the resource size, for the vmss it seems that SKU is populated and resource size is mapped in the name +func mapResourceSize(resource resources.GenericResource) string { + if resource.Sku != nil && resource.Sku.Name != nil { + return *resource.Sku.Name + } + return "" +} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go index 889c029a0e9d..ad1216cb7481 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go @@ -21,11 +21,16 @@ func MockResource() resources.GenericResource { name := "resourceName" location := "resourceLocation" rType := "resourceType" + skuName := "standard" + sku := resources.Sku{ + Name: &skuName, + } return resources.GenericResource{ ID: &id, Name: &name, Location: &location, Type: &rType, + Sku: &sku, } } @@ -93,15 +98,16 @@ func TestMapMetric(t *testing.T) { assert.Nil(t, err) assert.Equal(t, len(metrics), 2) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Id, "123") assert.Equal(t, metrics[1].Resource.Name, "resourceName") assert.Equal(t, metrics[1].Resource.Type, "resourceType") assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].Resource.Size, "standard") assert.Equal(t, metrics[1].Namespace, "namespace") assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go new file mode 100644 index 000000000000..05a125248899 --- /dev/null +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package compute_vm_scaleset + +import ( + "testing" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("compute_vm_scaleset") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("compute_vm_scaleset") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 2a2e1b41bb8b..1ca070ceba3c 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -32,7 +32,7 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err continue } // build a resource key with unique resource namespace combination - resNamkey := fmt.Sprintf("%s,%s", metric.Resource.ID, metric.Namespace) + resNamkey := fmt.Sprintf("%s,%s", metric.Resource.Id, metric.Namespace) groupByResourceNamespace[resNamkey] = append(groupByResourceNamespace[resNamkey], metric) } // grouping metrics by the dimensions configured @@ -124,7 +124,6 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) ModuleFields: common.MapStr{ "timegrain": metric.TimeGrain, "resource": common.MapStr{ - "name": metric.Resource.Name, "type": metric.Resource.Type, "group": metric.Resource.Group, }, @@ -150,7 +149,15 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") event.RootFields.Put("cloud.region", metric.Resource.Location) - + event.RootFields.Put("cloud.instance.name", metric.Resource.Name) + if metric.Resource.SubId != "" { + event.RootFields.Put("cloud.instance.id", metric.Resource.SubId) + } else { + event.RootFields.Put("cloud.instance.id", metric.Resource.Id) + } + if metric.Resource.Size != "" { + event.RootFields.Put("cloud.machine.type", metric.Resource.Size) + } metricList := common.MapStr{} for _, value := range metricValues { metricNameString := fmt.Sprintf("%s", managePropertyName(value.name)) diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go index ff1600607079..5f3f3dba630f 100644 --- a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go +++ b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go @@ -2,50 +2,38 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package database_account import ( - "errors" - "os" "testing" + "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("database_account") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"database_account"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("database_account") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/mock_service.go b/x-pack/metricbeat/module/azure/mock_service.go index 1d45cc6f8477..2df7f888a225 100644 --- a/x-pack/metricbeat/module/azure/mock_service.go +++ b/x-pack/metricbeat/module/azure/mock_service.go @@ -18,27 +18,33 @@ type MockService struct { mock.Mock } +// GetResourceDefinitionById is a mock function for the azure service +func (client *MockService) GetResourceDefinitionById(id string) (resources.GenericResource, error) { + args := client.Called(id) + return args.Get(0).(resources.GenericResource), args.Error(1) +} + // GetResourceDefinitions is a mock function for the azure service -func (client *MockService) GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) { - args := client.Called(ID, group, rType, query) +func (client *MockService) GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) { + args := client.Called(id, group, rType, query) return args.Get(0).(resources.ListResultPage), args.Error(1) } // GetMetricDefinitions is a mock function for the azure service -func (client *MockService) GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) { - args := client.Called(resourceID, namespace) +func (client *MockService) GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) { + args := client.Called(resourceId, namespace) return args.Get(0).(insights.MetricDefinitionCollection), args.Error(1) } // GetMetricNamespaces is a mock function for the azure service -func (client *MockService) GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) { - args := client.Called(resourceID) +func (client *MockService) GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) { + args := client.Called(resourceId) return args.Get(0).(insights.MetricNamespaceCollection), args.Error(1) } // GetMetricValues is a mock function for the azure service -func (client *MockService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { - args := client.Called(resourceID, namespace) +func (client *MockService) GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { + args := client.Called(resourceId, namespace) return args.Get(0).([]insights.Metric), args.String(1), args.Error(2) } diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/data.json b/x-pack/metricbeat/module/azure/monitor/_meta/data.json index d267245ce1e6..2bf28218d2d3 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/data.json +++ b/x-pack/metricbeat/module/azure/monitor/_meta/data.json @@ -1,58 +1,46 @@ { - "@timestamp":"2019-08-20T13:52:00.000Z", - "service":{ - "type":"azure" - }, - "azure":{ - "timegrain" : "PT1M", - "subscriptionID":"123456-qwer-1234-5678-12345678", - "metrics":{ - "percentage_cpu":{ - "min":1.86, - "max":2.8, - "avg":2.535 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "metrics": { + "datausage": { + "total": 0 + }, + "documentcount": { + "total": 0 + }, + "documentquota": { + "total": 53687091200 } }, - "resource":{ - "name":"vmwindows", - "type":"Microsoft.Compute/virtualMachines" + "namespace": "Microsoft.DocumentDb/databaseAccounts", + "resource": { + "group": "obs-infrastructure", + "tags": { + "defaultExperience": "Core (SQL)" + }, + "type": "Microsoft.DocumentDb/databaseAccounts" }, - "namespace":"Microsoft.Compute/virtualMachines" - }, - "cloud":{ - "provider":"azure", - "region":"westeurope" - }, - "metricset":{ - "name":"monitor" + "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "timegrain": "PT5M" }, - "event":{ - "module":"azure", - "duration":40189255100, - "dataset":"azure.monitor" + "cloud": { + "instance": { + "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", + "name": "obsaccount" + }, + "provider": "azure", + "region": "westeurope" }, - "ecs":{ - "version":"1.0.1" + "event": { + "dataset": "azure.monitor", + "duration": 115000, + "module": "azure" }, - "host":{ - "name":"DESKTOP-RFOOE09", - "architecture":"x86_64", - "os":{ - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.950 (WinBuild.160101.0800)", - "build":"17134.950", - "platform":"windows", - "version":"10.0" - }, - "id":"123456-qwer-1234-5678-12345678", - "hostname":"DESKTOP-RFOOE09" + "metricset": { + "name": "monitor", + "period": 10000 }, - "agent":{ - "ephemeral_id":"123456-qwer-1234-5678-12345678", - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 2a6ab1495e30..e94aebedceff 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -54,7 +54,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res for _, metricName := range metricGroup { metricNames = append(metricNames, *metricName.Name.Value) } - metrics = append(metrics, client.CreateMetric(*resource.ID, resource, metric.Namespace, metricNames, key, dim, metric.Timegrain)) + metrics = append(metrics, client.CreateMetric(*resource.ID, resource, "", metric.Namespace, metricNames, key, dim, metric.Timegrain)) } } } @@ -62,7 +62,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res } // filterMetricNames func will verify if the metric names entered are valid and will also return the corresponding list of metrics -func filterMetricNames(resourceID string, metricConfig azure.MetricConfig, metricDefinitions []insights.MetricDefinition) ([]string, error) { +func filterMetricNames(resourceId string, metricConfig azure.MetricConfig, metricDefinitions []insights.MetricDefinition) ([]string, error) { var supportedMetricNames []string var unsupportedMetricNames []string // check if all metric names are selected (*) @@ -75,11 +75,11 @@ func filterMetricNames(resourceID string, metricConfig azure.MetricConfig, metri supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions) if len(unsupportedMetricNames) > 0 { return nil, errors.Errorf("the metric names configured %s are not supported for the resource %s and namespace %s", - strings.Join(unsupportedMetricNames, ","), resourceID, metricConfig.Namespace) + strings.Join(unsupportedMetricNames, ","), resourceId, metricConfig.Namespace) } } if len(supportedMetricNames) == 0 { - return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceID, metricConfig.Namespace) + return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceId, metricConfig.Namespace) } return supportedMetricNames, nil } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index 06499e0be5a4..f3e46f3fe6dd 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -79,7 +79,7 @@ func TestMapMetric(t *testing.T) { resourceConfig.Metrics = []azure.MetricConfig{metricConfig} metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") @@ -100,7 +100,7 @@ func TestMapMetric(t *testing.T) { assert.Nil(t, err) assert.True(t, len(metrics) > 0) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") diff --git a/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go new file mode 100644 index 000000000000..2e022d83595f --- /dev/null +++ b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package monitor + +import ( + "testing" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("monitor") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("monitor") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + config["resources"] = []map[string]interface{}{{ + "resource_query": "resourceType eq 'Microsoft.DocumentDb/databaseAccounts'", + "metrics": []map[string]interface{}{{"namespace": "Microsoft.DocumentDb/databaseAccounts", + "name": []string{"DataUsage", "DocumentCount", "DocumentQuota"}}}}} + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index 50649704bd67..441442ac2020 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -29,16 +29,16 @@ type MonitorService struct { const metricNameLimit = 20 // NewService instantiates the Azure monitoring service -func NewService(clientID string, clientSecret string, tenantID string, subscriptionID string) (*MonitorService, error) { - clientConfig := auth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) +func NewService(clientId string, clientSecret string, tenantId string, subscriptionId string) (*MonitorService, error) { + clientConfig := auth.NewClientCredentialsConfig(clientId, clientSecret, tenantId) authorizer, err := clientConfig.Authorizer() if err != nil { return nil, err } - metricsClient := insights.NewMetricsClient(subscriptionID) - metricsDefinitionClient := insights.NewMetricDefinitionsClient(subscriptionID) - resourceClient := resources.NewClient(subscriptionID) - metricNamespaceClient := insights.NewMetricNamespacesClient(subscriptionID) + metricsClient := insights.NewMetricsClient(subscriptionId) + metricsDefinitionClient := insights.NewMetricDefinitionsClient(subscriptionId) + resourceClient := resources.NewClient(subscriptionId) + metricNamespaceClient := insights.NewMetricNamespacesClient(subscriptionId) metricsClient.Authorizer = authorizer metricsDefinitionClient.Authorizer = authorizer resourceClient.Authorizer = authorizer @@ -55,15 +55,15 @@ func NewService(clientID string, clientSecret string, tenantID string, subscript } // GetResourceDefinitions will retrieve the azure resources based on the options entered -func (service MonitorService) GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) { +func (service MonitorService) GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) { var resourceQuery string - if len(ID) > 0 { + if len(id) > 0 { var filterList []string // listing resourceID conditions does not seem to work with the API but querying by name or resource types will work - for _, id := range ID { - filterList = append(filterList, fmt.Sprintf("name eq '%s'", getResourceNameFromID(id))) + for _, id := range id { + filterList = append(filterList, fmt.Sprintf("name eq '%s'", getResourceNameFromId(id))) } - resourceQuery = fmt.Sprintf("(%s) AND resourceType eq '%s'", strings.Join(filterList, " OR "), getResourceTypeFromID(ID[0])) + resourceQuery = fmt.Sprintf("(%s) AND resourceType eq '%s'", strings.Join(filterList, " OR "), getResourceTypeFromId(id[0])) } else if len(group) > 0 { var filterList []string for _, gr := range group { @@ -76,21 +76,26 @@ func (service MonitorService) GetResourceDefinitions(ID []string, group []string } else if query != "" { resourceQuery = query } - return service.resourceClient.List(service.context, resourceQuery, "resourceID", nil) + return service.resourceClient.List(service.context, resourceQuery, "", nil) +} + +// GetResourceDefinitionById will retrieve the azure resource based on the resource Id +func (service MonitorService) GetResourceDefinitionById(id string) (resources.GenericResource, error) { + return service.resourceClient.GetByID(service.context, id) } // GetMetricNamespaces will return all supported namespaces based on the resource id and namespace -func (service *MonitorService) GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) { - return service.metricNamespaceClient.List(service.context, resourceID, "") +func (service *MonitorService) GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) { + return service.metricNamespaceClient.List(service.context, resourceId, "") } // GetMetricDefinitions will return all supported metrics based on the resource id and namespace -func (service *MonitorService) GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) { - return service.metricDefinitionClient.List(service.context, resourceID, namespace) +func (service *MonitorService) GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) { + return service.metricDefinitionClient.List(service.context, resourceId, namespace) } // GetMetricValues will return the metric values based on the resource and metric details -func (service *MonitorService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { +func (service *MonitorService) GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { var tg *string var interval string if timegrain != "" { @@ -103,12 +108,12 @@ func (service *MonitorService) GetMetricValues(resourceID string, namespace stri if end > len(metricNames) { end = len(metricNames) } - resp, err := service.metricsClient.List(service.context, resourceID, timespan, tg, strings.Join(metricNames[i:end], ","), + resp, err := service.metricsClient.List(service.context, resourceId, timespan, tg, strings.Join(metricNames[i:end], ","), aggregations, nil, "", filter, insights.Data, namespace) // check for applied charges before returning any errors if resp.Cost != nil && *resp.Cost != 0 { - service.log.Warnf("Charges amounted to %v are being applied while retrieving the metric values from the resource %s ", *resp.Cost, resourceID) + service.log.Warnf("Charges amounted to %v are being applied while retrieving the metric values from the resource %s ", *resp.Cost, resourceId) } if err != nil { return metrics, "", err diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index da51accc5029..c55757d24acc 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -11,15 +11,16 @@ import ( // Resource will contain the main azure resource details type Resource struct { - // SubID is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms - SubID string - ID string + // SubId is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms + SubId string + Id string Name string Location string Type string Group string Tags map[string]string Subscription string + Size string } // Metric will contain the main azure metric details diff --git a/x-pack/metricbeat/module/azure/service_interface.go b/x-pack/metricbeat/module/azure/service_interface.go index f8310b6ca7b9..e8985a7eedd8 100644 --- a/x-pack/metricbeat/module/azure/service_interface.go +++ b/x-pack/metricbeat/module/azure/service_interface.go @@ -11,8 +11,9 @@ import ( // Service interface for the azure monitor service and mock for testing type Service interface { - GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) - GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) - GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) - GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) + GetResourceDefinitionById(id string) (resources.GenericResource, error) + GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) + GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) + GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) + GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) } diff --git a/x-pack/metricbeat/module/azure/storage/_meta/data.json b/x-pack/metricbeat/module/azure/storage/_meta/data.json index 6ede71f6edce..599467a3e3ce 100644 --- a/x-pack/metricbeat/module/azure/storage/_meta/data.json +++ b/x-pack/metricbeat/module/azure/storage/_meta/data.json @@ -1,85 +1,46 @@ { - "_index" : "metricbeat-8.0.0-2020.01.09-000001", - "_type" : "_doc", - "_id" : "XxYjjG8BQe78yNOEwING", - "_score" : null, - "_source" : { - "@timestamp" : "2020-01-09T21:04:00.000Z", - "event" : { - "duration" : 122304420600, - "dataset" : "azure.storage", - "module" : "azure" - }, - "metricset" : { - "name" : "storage", - "period" : 300000 - }, - "azure" : { - "timegrain" : "PT5M", - "namespace" : "Microsoft.Storage/storageAccounts/blobServices", - "dimensions" : { - "apiname" : "DeleteBlob" - }, - "storage" : { - "successe2elatency" : { - "avg" : 9.1 - }, - "availability" : { - "avg" : 100 - }, - "transactions" : { - "total" : 10 - }, - "ingress" : { - "total" : 5330 - }, - "egress" : { - "total" : 1010 - }, - "successserverlatency" : { - "avg" : 9.1 - } + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "namespace": "Microsoft.Storage/storageAccounts/queueServices", + "resource": { + "group": "cloud-shell-storage-westus", + "tags": { + "ms-resource-usage": "azure-cloud-shell" }, - "resource" : { - "name" : "obsdiaglinux", - "type" : "Microsoft.Storage/storageAccounts", - "group" : "obs-infrastructure" - }, - "subscription_id" : "12345678-qwer-1234-5678-12345678" - }, - "service" : { - "type" : "azure" + "type": "Microsoft.Storage/storageAccounts" }, - "ecs" : { - "version" : "1.2.0" - }, - "host" : { - "architecture" : "x86_64", - "name" : "DESKTOP-RFOOE09", - "os" : { - "kernel" : "10.0.18362.535 (WinBuild.160101.0800)", - "build" : "18362.535", - "platform" : "windows", - "version" : "10.0", - "family" : "windows", - "name" : "Windows 10 Pro" + "storage": { + "queuecapacity": { + "avg": 0 + }, + "queuecount": { + "avg": 0 }, - "id" : "12345678-qwer-1234-5678-12345678", - "hostname" : "DESKTOP-RFOOE09" + "queuemessagecount": { + "avg": 0 + } }, - "agent" : { - "hostname" : "DESKTOP-RFOOE09", - "id" : "12345678-qwer-1234-5678-12345678", - "version" : "8.0.0", - "type" : "metricbeat", - "ephemeral_id" : "12345678-qwer-1234-5678-12345678" + "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "timegrain": "PT1H" + }, + "cloud": { + "instance": { + "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/cloud-shell-storage-westus/providers/Microsoft.Storage/storageAccounts/strAc/queueServices/default", + "name": "strAc" }, - "cloud" : { - "provider" : "azure", - "region" : "westeurope" - } + "provider": "azure", + "region": "westus" + }, + "event": { + "dataset": "azure.storage", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "storage", + "period": 10000 }, - "sort" : [ - 1578603840000 - ] + "service": { + "type": "azure" + } } diff --git a/x-pack/metricbeat/module/azure/storage/client_helper.go b/x-pack/metricbeat/module/azure/storage/client_helper.go index 488f90d21ebd..3a6f69315b09 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper.go @@ -61,7 +61,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if dimension != azure.NoDimension { dimensions = []azure.Dimension{{Name: dimension, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, mets, resource, resourceID, namespace, dimensions, time)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(mets, resource, resourceID, "", namespace, dimensions, time)...) } } } diff --git a/x-pack/metricbeat/module/azure/storage/client_helper_test.go b/x-pack/metricbeat/module/azure/storage/client_helper_test.go index f9dd066df33e..88f0eb211f03 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper_test.go @@ -109,12 +109,12 @@ func TestMapMetric(t *testing.T) { client.AzureMonitorService = m metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") assert.Equal(t, metrics[0].Namespace, "Microsoft.Storage/storageAccounts") - assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Id, "123") assert.Equal(t, metrics[1].Resource.Name, "resourceName") assert.Equal(t, metrics[1].Resource.Type, "resourceType") assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") diff --git a/x-pack/metricbeat/module/azure/storage/storage_integration_test.go b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go new file mode 100644 index 000000000000..39e35f9c691d --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package storage + +import ( + "testing" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("storage") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("storage") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/test/integration.go b/x-pack/metricbeat/module/azure/test/integration.go new file mode 100644 index 000000000000..8f2b03851140 --- /dev/null +++ b/x-pack/metricbeat/module/azure/test/integration.go @@ -0,0 +1,36 @@ +package test + +import ( + "errors" + "os" +) + +// GetConfig function gets azure credentials for integration tests. +func GetConfig(metricSetName string) (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("could not find var AZURE_CLIENT_ID") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("could not find var AZURE_CLIENT_SECRET") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("could not find var AZURE_TENANT_ID") + } + subId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("could not find var AZURE_SUBSCRIPTION_ID") + } + return map[string]interface{}{ + "module": "azure", + "period": "300s", + "refresh_list_interval": "600s", + "metricsets": []string{metricSetName}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subId, + }, nil +} From a97cbc41c76338ee3d7d58941f416886358e235c Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 11:41:18 +0100 Subject: [PATCH 2/7] mage fmt --- CHANGELOG.next.asciidoc | 1 + x-pack/metricbeat/module/azure/test/integration.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 98a30512188f..6792f1efdc83 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -194,6 +194,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] +- Align fields to ECS and add more tests for the azure module. {issue}16024[16024] {pull}16754[16754] *Packetbeat* diff --git a/x-pack/metricbeat/module/azure/test/integration.go b/x-pack/metricbeat/module/azure/test/integration.go index 8f2b03851140..9e8e011a217c 100644 --- a/x-pack/metricbeat/module/azure/test/integration.go +++ b/x-pack/metricbeat/module/azure/test/integration.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package test import ( From 90e2b72bee7e68e4fb68fb69501371fdd2c773af Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 13:27:09 +0100 Subject: [PATCH 3/7] fix --- x-pack/metricbeat/module/azure/azure.go | 2 +- .../azure/compute_vm_scaleset/client_helper.go | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 3a4531b77201..b321ac5f8aeb 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -106,7 +106,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription var resources []ResourceConfig for _, resource := range config.Resources { - if hasConfigOptions(resource.Group) || hasConfigOptions(resource.ID) { + if hasConfigOptions(resource.Group) || hasConfigOptions(resource.Id) { resources = append(resources, resource) } } diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index 15b963557c9f..f45896dce8eb 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -77,23 +77,13 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if key != azure.NoDimension { dimensions = []azure.Dimension{{Name: key, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, azure.DefaultTimeGrain)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(metricGroup, resource, "", resourceSize, metric.Namespace, dimensions, azure.DefaultTimeGrain)...) } } } return metrics, nil } -// containsDimension will check if the dimension value is found in the list -func containsDimension(dimension string, dimensions []insights.LocalizableString) bool { - for _, dim := range dimensions { - if *dim.Value == dimension { - return true - } - } - return false -} - // mapResourceSize func will try to map if existing the resource size, for the vmss it seems that SKU is populated and resource size is mapped in the name func mapResourceSize(resource resources.GenericResource) string { if resource.Sku != nil && resource.Sku.Name != nil { From 5bd58bdda2aa6d305728c11e68aa32859104d97c Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 15:18:59 +0100 Subject: [PATCH 4/7] work on tests --- metricbeat/docs/fields.asciidoc | 10 - .../metricbeat/module/azure/_meta/fields.yml | 4 - .../module/azure/compute_vm/_meta/data.json | 554 ++++++++++++++++-- .../compute_vm/compute_vm_integration_test.go | 4 +- .../azure/compute_vm_scaleset/_meta/data.json | 87 +-- .../compute_vm_scaleset_integration_test.go | 4 +- .../container_instance_integration_test.go | 56 +- .../container_instance_test.go | 2 +- .../container_registry_integration_test.go | 56 +- .../container_registry_test.go | 2 +- .../container_service_integration_test.go | 56 +- .../container_service_test.go | 2 +- x-pack/metricbeat/module/azure/data_test.go | 2 +- .../database_account_integration_test.go | 4 +- x-pack/metricbeat/module/azure/fields.go | 2 +- .../module/azure/monitor/_meta/data.json | 10 +- .../azure/monitor/monitor_integration_test.go | 4 +- .../module/azure/storage/_meta/data.json | 19 +- .../azure/storage/storage_integration_test.go | 4 +- 19 files changed, 599 insertions(+), 283 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4cb142e1ee7e..63a504663207 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3123,16 +3123,6 @@ The resource specified -*`azure.resource.name`*:: -+ --- -The name of the resource - - -type: keyword - --- - *`azure.resource.type`*:: + -- diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml index 4d51e8240876..61913ce00199 100644 --- a/x-pack/metricbeat/module/azure/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/_meta/fields.yml @@ -17,10 +17,6 @@ description: > The resource specified fields: - - name: name - type: keyword - description: > - The name of the resource - name: type type: keyword description: > diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 4a6319ef319d..5858cf0407e0 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -2,133 +2,557 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm": { - "data_disk_queue_depth": { + "asp_net_application_restarts": { "avg": 0 }, - "data_disk_read_bytes_per_sec": { + "asp_net_applications_anonymous_requests": { "avg": 0 }, - "data_disk_read_operations_per_sec": { + "asp_net_applications_anonymous_requests_per_sec": { "avg": 0 }, - "data_disk_write_bytes_per_sec": { + "asp_net_applications_cache_api_entries": { "avg": 0 }, - "data_disk_write_operations_per_sec": { + "asp_net_applications_cache_api_hit_ratio": { "avg": 0 }, - "disk_read_bytes": { - "total": 32768.02 + "asp_net_applications_cache_api_hits": { + "avg": 0 + }, + "asp_net_applications_cache_api_misses": { + "avg": 0 + }, + "asp_net_applications_cache_api_turnover_rate": { + "avg": 0 + }, + "asp_net_applications_cache_total_entries": { + "avg": 0 + }, + "asp_net_applications_cache_total_hit_ratio": { + "avg": 0 + }, + "asp_net_applications_cache_total_hits": { + "avg": 0 + }, + "asp_net_applications_cache_total_misses": { + "avg": 0 + }, + "asp_net_applications_cache_total_turnover_rate": { + "avg": 0 + }, + "asp_net_applications_compilations_total": { + "avg": 0 + }, + "asp_net_applications_debugging_requests": { + "avg": 0 + }, + "asp_net_applications_errors_during_compilation": { + "avg": 0 + }, + "asp_net_applications_errors_during_execution": { + "avg": 0 + }, + "asp_net_applications_errors_during_preprocessing": { + "avg": 0 + }, + "asp_net_applications_errors_total": { + "avg": 0 + }, + "asp_net_applications_errors_total_per_sec": { + "avg": 0 + }, + "asp_net_applications_errors_unhandled_during_execution": { + "avg": 0 + }, + "asp_net_applications_errors_unhandled_during_execution_per_sec": { + "avg": 0 + }, + "asp_net_applications_output_cache_entries": { + "avg": 0 + }, + "asp_net_applications_output_cache_hit_ratio": { + "avg": 0 + }, + "asp_net_applications_output_cache_hits": { + "avg": 0 + }, + "asp_net_applications_output_cache_misses": { + "avg": 0 + }, + "asp_net_applications_output_cache_turnover_rate": { + "avg": 0 + }, + "asp_net_applications_pipeline_instance_count": { + "avg": 0 + }, + "asp_net_applications_request_bytes_in_total": { + "avg": 0 + }, + "asp_net_applications_request_bytes_out_total": { + "avg": 0 + }, + "asp_net_applications_requests_executing": { + "avg": 0 + }, + "asp_net_applications_requests_failed": { + "avg": 0 + }, + "asp_net_applications_requests_in_application_queue": { + "avg": 0 + }, + "asp_net_applications_requests_not_authorized": { + "avg": 0 + }, + "asp_net_applications_requests_not_found": { + "avg": 0 + }, + "asp_net_applications_requests_per_sec": { + "avg": 0 + }, + "asp_net_applications_requests_succeeded": { + "avg": 0 + }, + "asp_net_applications_requests_timed_out": { + "avg": 0 + }, + "asp_net_applications_requests_total": { + "avg": 0 + }, + "asp_net_applications_running": { + "avg": 0 + }, + "asp_net_applications_sessions_abandoned": { + "avg": 0 + }, + "asp_net_applications_sessions_active": { + "avg": 0 + }, + "asp_net_applications_sessions_timed_out": { + "avg": 0 + }, + "asp_net_applications_sessions_total": { + "avg": 0 + }, + "asp_net_applications_transactions_aborted": { + "avg": 0 + }, + "asp_net_applications_transactions_committed": { + "avg": 0 + }, + "asp_net_applications_transactions_pending": { + "avg": 0 + }, + "asp_net_applications_transactions_per_sec": { + "avg": 0 + }, + "asp_net_applications_transactions_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_anonymous_requests": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_anonymous_requests_per_sec": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_entries": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_hit_ratio": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_hits": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_misses": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_turnover_rate": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_entries": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_hit_ratio": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_hits": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_misses": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_turnover_rate": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_compilations_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_debugging_requests": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_during_compilation": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_during_execution": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_during_preprocessing": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_total_per_sec": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_unhandled_during_execution": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_unhandled_during_execution_per_sec": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_entries": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_hit_ratio": { + "avg": 0 }, - "disk_read_operations_per_sec": { - "avg": 0.008 + "asp_net_apps_v4_0_30319_output_cache_hits": { + "avg": 0 }, - "disk_write_bytes": { - "total": 19635551.44 + "asp_net_apps_v4_0_30319_output_cache_misses": { + "avg": 0 }, - "disk_write_operations_per_sec": { - "avg": 2.8600000000000003 + "asp_net_apps_v4_0_30319_output_cache_turnover_rate": { + "avg": 0 }, - "inbound_flows": { - "avg": 52.2 + "asp_net_apps_v4_0_30319_pipeline_instance_count": { + "avg": 0 }, - "inbound_flows_maximum_creation_rate": { - "avg": 11.6 + "asp_net_apps_v4_0_30319_request_bytes_in_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_request_bytes_out_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_requests_executing": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_requests_failed": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_requests_in_application_queue": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_requests_not_authorized": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_requests_not_found": { + "avg": 0 }, - "network_in": { - "total": 161463 + "asp_net_apps_v4_0_30319_requests_per_sec": { + "avg": 0 }, - "network_in_total": { - "total": 17721837 + "asp_net_apps_v4_0_30319_requests_succeeded": { + "avg": 0 }, - "network_out": { - "total": 129547 + "asp_net_apps_v4_0_30319_requests_timed_out": { + "avg": 0 }, - "network_out_total": { - "total": 1740212 + "asp_net_apps_v4_0_30319_requests_total": { + "avg": 0 }, - "os_disk_queue_depth": { - "avg": 0.01 + "asp_net_apps_v4_0_30319_sessions_abandoned": { + "avg": 0 }, - "os_disk_read_bytes_per_sec": { - "avg": 109.226 + "asp_net_apps_v4_0_30319_sessions_active": { + "avg": 0 }, - "os_disk_read_operations_per_sec": { - "avg": 0.008 + "asp_net_apps_v4_0_30319_sessions_timed_out": { + "avg": 0 }, - "os_disk_write_bytes_per_sec": { - "avg": 65451.831999999995 + "asp_net_apps_v4_0_30319_sessions_total": { + "avg": 0 }, - "os_disk_write_operations_per_sec": { - "avg": 2.8600000000000003 + "asp_net_apps_v4_0_30319_transactions_aborted": { + "avg": 0 }, - "os_per_disk_qd": { - "avg": 0.01 + "asp_net_apps_v4_0_30319_transactions_committed": { + "avg": 0 }, - "os_per_disk_read_bytes_per_sec": { - "avg": 109.226 + "asp_net_apps_v4_0_30319_transactions_pending": { + "avg": 0 }, - "os_per_disk_read_operations_per_sec": { - "avg": 0.008 + "asp_net_apps_v4_0_30319_transactions_per_sec": { + "avg": 0 }, - "os_per_disk_write_bytes_per_sec": { - "avg": 65451.831999999995 + "asp_net_apps_v4_0_30319_transactions_total": { + "avg": 0 }, - "os_per_disk_write_operations_per_sec": { - "avg": 2.8600000000000003 + "asp_net_request_execution_time": { + "avg": 0 }, - "outbound_flows": { - "avg": 52.2 + "asp_net_request_wait_time": { + "avg": 0 }, - "outbound_flows_maximum_creation_rate": { - "avg": 11.6 + "asp_net_requests_current": { + "avg": 0 }, - "per_disk_qd": { + "asp_net_requests_disconnected": { "avg": 0 }, - "per_disk_read_bytes_per_sec": { + "asp_net_requests_queued": { "avg": 0 }, - "per_disk_read_operations_per_sec": { + "asp_net_requests_rejected": { "avg": 0 }, - "per_disk_write_bytes_per_sec": { + "asp_net_v4_0_30319_application_restarts": { "avg": 0 }, - "per_disk_write_operations_per_sec": { + "asp_net_v4_0_30319_applications_running": { "avg": 0 }, - "percentage_cpu": { - "avg": 2.489 + "asp_net_v4_0_30319_request_execution_time": { + "avg": 0 }, - "premium_os_disk_cache_read_hit": { + "asp_net_v4_0_30319_request_wait_time": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_current": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_disconnected": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_queued": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_rejected": { + "avg": 0 + }, + "asp_net_v4_0_30319_worker_process_restarts": { + "avg": 0 + }, + "asp_net_v4_0_30319_worker_processes_running": { + "avg": 0 + }, + "asp_net_worker_process_restarts": { + "avg": 0 + }, + "asp_net_worker_processes_running": { + "avg": 0 + }, + "logical_disk_avg_disk_queue_length": { + "avg": 0 + }, + "logical_disk_avg_disk_read_queue_length": { + "avg": 0 + }, + "logical_disk_avg_disk_sec_per_read": { + "avg": 0 + }, + "logical_disk_avg_disk_sec_per_transfer": { + "avg": 0 + }, + "logical_disk_avg_disk_sec_per_write": { + "avg": 0 + }, + "logical_disk_avg_disk_write_queue_length": { + "avg": 0 + }, + "logical_disk_disk_bytes_per_sec": { + "avg": 449380 + }, + "logical_disk_disk_read_bytes_per_sec": { + "avg": 4783 + }, + "logical_disk_disk_reads_per_sec": { + "avg": 0 + }, + "logical_disk_disk_transfers_per_sec": { + "avg": 6.6 + }, + "logical_disk_disk_write_bytes_per_sec": { + "avg": 444596.8 + }, + "logical_disk_disk_writes_per_sec": { + "avg": 6.4 + }, + "logical_disk_free_megabytes": { + "avg": 10433 + }, + "logical_disk_pct_disk_read_time": { + "avg": 0 + }, + "logical_disk_pct_disk_time": { + "avg": 1.6 + }, + "logical_disk_pct_disk_write_time": { + "avg": 1.6 + }, + "logical_disk_pct_free_space": { + "avg": 22 + }, + "logical_disk_pct_idle_time": { + "avg": 98.6 + }, + "memory_available_bytes": { + "avg": 2951829094.4 + }, + "memory_cache_bytes": { + "avg": 60126003.2 + }, + "memory_committed_bytes": { + "avg": 8922399539.2 + }, + "memory_page_faults_per_sec": { + "avg": 1484.2 + }, + "memory_pages_per_sec": { + "avg": 6 + }, + "memory_pct_committed_bytes_in_use": { + "avg": 82.4 + }, + "memory_pool_nonpaged_bytes": { + "avg": 204028313.6 + }, + "memory_pool_paged_bytes": { + "avg": 359613235.2 + }, + "network_interface_bytes_received_per_sec": { + "avg": 7922.333333333333 + }, + "network_interface_bytes_sent_per_sec": { + "avg": 60510.666666666664 + }, + "network_interface_bytes_total_per_sec": { + "avg": 68433.26666666666 + }, + "network_interface_packets_outbound_errors": { + "avg": 0 + }, + "network_interface_packets_per_sec": { + "avg": 43.4 + }, + "network_interface_packets_received_errors": { + "avg": 0 + }, + "network_interface_packets_received_per_sec": { + "avg": 27.533333333333335 + }, + "network_interface_packets_sent_per_sec": { + "avg": 15.733333333333333 + }, + "process_handle_count": { + "avg": 119553 + }, + "process_thread_count": { + "avg": 1705.8 + }, + "process_working_set": { + "avg": 5566961254.4 + }, + "process_working_set_private": { + "avg": 4168472985.6 + }, + "processor_information_pct_privileged_time": { + "avg": 2.6 + }, + "processor_information_pct_processor_time": { + "avg": 6.4 + }, + "processor_information_pct_user_time": { + "avg": 3.4 + }, + "processor_information_processor_frequency": { + "avg": 2295 + }, + "sqlserver_buffer_manager_buffer_cache_hit_ratio": { "avg": 100 }, - "premium_os_disk_cache_read_miss": { + "sqlserver_buffer_manager_checkpoint_pages_per_sec": { + "avg": 0 + }, + "sqlserver_buffer_manager_database_pages": { + "avg": 1483 + }, + "sqlserver_buffer_manager_lazy_writes_per_sec": { + "avg": 0 + }, + "sqlserver_buffer_manager_page_reads_per_sec": { + "avg": 0 + }, + "sqlserver_buffer_manager_page_writes_per_sec": { + "avg": 0 + }, + "sqlserver_general_statistics_user_connections": { + "avg": 1 + }, + "sqlserver_memory_manager_memory_grants_pending": { + "avg": 0 + }, + "sqlserver_memory_manager_total_server_memory": { + "avg": 317976 + }, + "sqlserver_sql_statistics_batch_requests_per_sec": { + "avg": 0 + }, + "sqlserver_sql_statistics_sql_compilations_per_sec": { + "avg": 0 + }, + "sqlserver_sql_statistics_sql_re-compilations_per_sec": { + "avg": 0 + }, + "system_context_switches_per_sec": { + "avg": 7782.6 + }, + "system_processes": { + "avg": 114 + }, + "system_processor_queue_length": { + "avg": 0.2 + }, + "system_system_up_time": { + "avg": 12530764 + }, + "web_service_bytes_total_per_sec": { + "avg": 0 + }, + "web_service_current_connections": { + "avg": 0 + }, + "web_service_isapi_extension_requests_per_sec": { + "avg": 0 + }, + "web_service_total_method_requests_per_sec": { "avg": 0 } }, - "namespace": "Microsoft.Compute/virtualMachines", + "namespace": "Azure.VM.Windows.GuestMetrics", "resource": { "group": "obs-infrastructure", "tags": { - "provider": "648D2193-0CE0-4EFB-8A82-AF9792184FD9" + "vmtest": "value1, value 2", + "vmtest1": "value3" }, "type": "Microsoft.Compute/virtualMachines" }, - "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { "instance": { - "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/elkdata-0", - "name": "elkdata-0" + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", + "name": "obstestmemleak" }, "machine": { - "type": "Standard_DS1_v2" + "type": "Standard_B2ms" }, "provider": "azure", - "region": "eastus" + "region": "westeurope" }, "event": { "dataset": "azure.compute_vm", diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go index 107becd51613..043194052078 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go @@ -9,11 +9,11 @@ package compute_vm import ( "testing" - "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" "github.com/stretchr/testify/assert" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index 405c0c740e44..8edb9c5b69ac 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -2,101 +2,46 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm_scaleset": { - "data_disk_queue_depth": { - "avg": 0 - }, - "data_disk_read_bytes_per_sec": { - "avg": 0 - }, - "data_disk_read_operations_per_sec": { - "avg": 0 - }, - "data_disk_write_bytes_per_sec": { - "avg": 0 - }, - "data_disk_write_operations_per_sec": { - "avg": 0 - }, - "disk_read_bytes": { - "total": 0 - }, - "disk_read_operations_per_sec": { - "avg": 0 + "cpu_credits_consumed": { + "avg": 0.019 }, - "disk_write_bytes": { - "total": 1180450.5 + "cpu_credits_remaining": { + "avg": 97.99 }, - "disk_write_operations_per_sec": { - "avg": 0.6839999999999999 - }, - "inbound_flows": { - "avg": 36.8 - }, - "inbound_flows_maximum_creation_rate": { - "avg": 6.6 - }, - "network_in": { - "total": 1158350 - }, - "network_in_total": { - "total": 1737301 - }, - "network_out": { - "total": 175875 - }, - "network_out_total": { - "total": 646949 - }, - "os_disk_queue_depth": { + "os_per_disk_qd": { "avg": 0 }, - "os_disk_read_bytes_per_sec": { + "os_per_disk_read_bytes_per_sec": { "avg": 0 }, - "os_disk_read_operations_per_sec": { + "os_per_disk_read_operations_per_sec": { "avg": 0 }, - "os_disk_write_bytes_per_sec": { - "avg": 3934.8300000000004 - }, - "os_disk_write_operations_per_sec": { - "avg": 0.6839999999999999 - }, - "outbound_flows": { - "avg": 37 + "os_per_disk_write_bytes_per_sec": { + "avg": 19201.653 }, - "outbound_flows_maximum_creation_rate": { - "avg": 7.6 - }, - "percentage_cpu": { - "avg": 0.30125 + "os_per_disk_write_operations_per_sec": { + "avg": 1.0366666666666666 } }, - "dimensions": { - "vmname": "ecescaletest_3" - }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { "group": "obs-infrastructure", - "tags": { - "environment": "staging", - "role": "allocator" - }, "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { "instance": { - "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/ecescaletest", - "name": "ecescaletest" + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss" }, "machine": { - "type": "Standard_D4s_v3" + "type": "Standard_B1ls" }, "provider": "azure", - "region": "eastus2" + "region": "westeurope" }, "event": { "dataset": "azure.compute_vm_scaleset", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go index 05a125248899..acd6450a97b6 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go @@ -9,11 +9,11 @@ package compute_vm_scaleset import ( "testing" - "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" "github.com/stretchr/testify/assert" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go index 37b814b070f9..0907609f3231 100644 --- a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go @@ -2,50 +2,38 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package container_instance import ( - "errors" - "os" "testing" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("container_instance") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"container_instance"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("container_instance") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go index a92237aea4bd..3b91ef661662 100644 --- a/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go @@ -7,7 +7,7 @@ package container_instance import ( "os" - "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb" ) func init() { diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go index 41414529997b..f47f6ef2ca5e 100644 --- a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go @@ -2,50 +2,38 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package container_registry import ( - "errors" - "os" "testing" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("container_registry") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"container_registry"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("container_registry") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go index 7b65c68056fe..6c94aab4345f 100644 --- a/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go @@ -7,7 +7,7 @@ package container_registry import ( "os" - "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb" ) func init() { diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go index 01c9f5a053e1..baa6be30d55b 100644 --- a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go @@ -2,50 +2,38 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package container_service import ( - "errors" - "os" "testing" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("container_service") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"container_registry"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("container_service") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_test.go index 0fb0b53347e9..33b9513348fc 100644 --- a/x-pack/metricbeat/module/azure/container_service/container_service_test.go +++ b/x-pack/metricbeat/module/azure/container_service/container_service_test.go @@ -7,7 +7,7 @@ package container_service import ( "os" - "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb" ) func init() { diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index 2498d92a5584..d26f9270f0f7 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common" ) func TestReturnAllDimensions(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go index 5f3f3dba630f..8099e7296871 100644 --- a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go +++ b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go @@ -9,11 +9,11 @@ package database_account import ( "testing" - "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" "github.com/stretchr/testify/assert" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 25f8d1b6fccd..478d649d8a21 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUlr+O2zAMxvc8BZExwOUBPBQo0KVDt+6GIjMue7ZkkPQV6dMXjm2d/D/p3ZBoCBBJ/r7fJ9KCX+AVLwmYvzXjDkBJC0xgf/2/3wFkKJapUvIugS87AGj3QumzumgeYSzQCCZwQjU7gDNhkUly3foCzpT4Lt8MvVSYQM6+rrqZGY+hTCylVGLOhlxY6SVf8fLHcxbNzwq34+cvhK9tEFQmO6PbOzKKr9nixDDOcINdrwNSoaUzYYw6jhsDNL+DheXEGxg9SqMI/gwaYc1aNz6fa908fpP1+HQ/wTtUYKodAptcjodZW3/6jVZHS+1kugYWbUlLU1Xk8m7//rC/L0TbsSHGFXbSr82vVGamYe9+Q4IUCBZoNWrZ3k3qU5BIKfu4ZywI379NDDMq0Ql5N6zTQo026nNrbVaYB7dIBDcBt76sasX0rTweBujjG/SeSOfCm4XF/w30zrkSIRVrChTUZ8kSgMP6TDqnhhxySk7UOIuPHq7jhZ53JRJjTqJ8eZZIPe9KJEF+o+cpUoc7vdCMmpMRTI21vnYP/kb1tNDRTuKU3pF63vxaWsq14j2VXvtsaq9kGR3nytFtHt89R7gRpRk/WkBg1JodZsflq0nUs8kfvNU7yNAZ/wIAAP//ilZVQA==" + return "eJzUls+OmzAQxu95ilGOkTYPwKFSpV566K135JgJnS7YaGbYij59RQCv+b9pe0h8iBTbfN/38wwWL/CKTQLmd814AFDSAhM43v4fDwAZimWqlLxL4NMBALq9UPqsLtpHGAs0gglcUM0B4EpYZJLctr6AMyW+y7dDmwoTyNnXVT+z4DGWiaWUSszZkAsrg+QrNr88Z9H8onA3vv9A+NyBoDLZBd3BkVF8zRZnhjHDB+wGHZAKLV0J46hT3BFyU+FoYZ14J8YQpX0c/BU0irVoPUX8D97hGObaAdjkcj4t2vrLT7Q6Weom061g0Za0NFVFLu/3H0/H+yC6tgkYt7Czpml/pTILXXN3mwYpECzQatQ3g5vUlyCRUvbvnrEgfP0yM8yoRCfk3bhOKzXaqc9Ha7ORefQqR+Fmwa0vq1oxfSvPp1H06TV2D9K18GZl8W+B3nNuIKRiTYGC+iwsIXBYX6Bzasghp+REjbP46HB9XhjybiAx5iTKzbMgDXk3kAT5jZ6nSH3c+YVm1FyMYGqs9bV78DdqSAt92hlO6R2p591PljWuDe+59Na3S3cly+Q4N45u9/juOcIdlHZ86wICo9bsMDuvX02ink3+4K3ehwyd8ScAAP//n2w3Eg==" } diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/data.json b/x-pack/metricbeat/module/azure/monitor/_meta/data.json index 2bf28218d2d3..67f5b91b62be 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/data.json +++ b/x-pack/metricbeat/module/azure/monitor/_meta/data.json @@ -2,13 +2,13 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "metrics": { - "datausage": { + "data_usage": { "total": 0 }, - "documentcount": { + "document_count": { "total": 0 }, - "documentquota": { + "document_quota": { "total": 53687091200 } }, @@ -20,12 +20,12 @@ }, "type": "Microsoft.DocumentDb/databaseAccounts" }, - "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { "instance": { - "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", "name": "obsaccount" }, "provider": "azure", diff --git a/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go index 2e022d83595f..87de0e38bce7 100644 --- a/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go +++ b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go @@ -9,11 +9,11 @@ package monitor import ( "testing" - "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" "github.com/stretchr/testify/assert" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/storage/_meta/data.json b/x-pack/metricbeat/module/azure/storage/_meta/data.json index 599467a3e3ce..76700ecb8d5b 100644 --- a/x-pack/metricbeat/module/azure/storage/_meta/data.json +++ b/x-pack/metricbeat/module/azure/storage/_meta/data.json @@ -3,33 +3,30 @@ "azure": { "namespace": "Microsoft.Storage/storageAccounts/queueServices", "resource": { - "group": "cloud-shell-storage-westus", - "tags": { - "ms-resource-usage": "azure-cloud-shell" - }, + "group": "obs-infrastructure", "type": "Microsoft.Storage/storageAccounts" }, "storage": { - "queuecapacity": { + "queue_capacity": { "avg": 0 }, - "queuecount": { + "queue_count": { "avg": 0 }, - "queuemessagecount": { + "queue_message_count": { "avg": 0 } }, - "subscription_id": "271dc764-105f-4af9-b324-d8190e40e5eb", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT1H" }, "cloud": { "instance": { - "id": "/subscriptions/271dc764-105f-4af9-b324-d8190e40e5eb/resourceGroups/cloud-shell-storage-westus/providers/Microsoft.Storage/storageAccounts/strAc/queueServices/default", - "name": "strAc" + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Storage/storageAccounts/urcbyscmrkbygsawinvm/queueServices/default", + "name": "urcbyscmrkbygsawinvm" }, "provider": "azure", - "region": "westus" + "region": "westeurope" }, "event": { "dataset": "azure.storage", diff --git a/x-pack/metricbeat/module/azure/storage/storage_integration_test.go b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go index 39e35f9c691d..920f21393c16 100644 --- a/x-pack/metricbeat/module/azure/storage/storage_integration_test.go +++ b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go @@ -9,11 +9,11 @@ package storage import ( "testing" - "github.com/elastic/beats/x-pack/metricbeat/module/azure/test" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" "github.com/stretchr/testify/assert" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) func TestFetchMetricset(t *testing.T) { From b416523ec5f320f80547ef0ce8360e87c07f75e0 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 16:30:43 +0100 Subject: [PATCH 5/7] fix --- x-pack/metricbeat/module/azure/data_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index d26f9270f0f7..1ef042b12c2e 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -75,7 +75,7 @@ func TestCreateEvent(t *testing.T) { } metric := Metric{ Resource: Resource{ - ID: "resId", + Id: "resId", Name: "res", Location: "west_europe", Type: "resType", From e7723b4ad7b6866c00ec31f3378f3ae33325476f Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 19:08:03 +0100 Subject: [PATCH 6/7] work on tests --- .../azure/container_instance/_meta/data.json | 74 +++++++++---------- .../container_instance_integration_test.go | 3 + .../container_registry_integration_test.go | 3 + .../azure/container_service/_meta/data.json | 72 +++++++++--------- .../container_service_integration_test.go | 3 + 5 files changed, 79 insertions(+), 76 deletions(-) diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json index 163b491021ed..aec7b3bdffba 100644 --- a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json @@ -1,46 +1,46 @@ { - "@timestamp" : "2020-02-28T14:20:00.000Z", - "cloud" : { - "provider" : "azure", - "region" : "westeurope" - }, - "ecs" : { - "version" : "1.4.0" - }, - "metricset" : { - "period" : 300000, - "name" : "container_instance" - }, - "event" : { - "duration" : 2383878000, - "dataset" : "azure.container_instance", - "module" : "azure" - }, - "azure" : { - "dimensions" : { - "container_name" : "testcontainergroup" - }, - "container_instance" : { - "cpu_usage" : { - "avg" : 0 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "container_instance": { + "cpu_usage": { + "avg": 0 }, - "memory_usage" : { - "avg" : 0 + "memory_usage": { + "avg": 0 } }, - "timegrain" : "PT5M", - "resource" : { - "group" : "obs-infrastructure", - "tags" : { - "tag1" : "value1" + "dimensions": { + "container_name": "testcontainergroup" + }, + "namespace": "Microsoft.ContainerInstance/containerGroups", + "resource": { + "group": "obs-infrastructure", + "tags": { + "tag1": "value1" }, - "name" : "testcontainergroup", - "type" : "Microsoft.ContainerInstance/containerGroups" + "type": "Microsoft.ContainerInstance/containerGroups" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" + }, + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerInstance/containerGroups/testcontainergroup", + "name": "testcontainergroup" }, - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerInstance/containerGroups" + "provider": "azure", + "region": "westeurope" + }, + "event": { + "dataset": "azure.container_instance", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "container_instance", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go index 0907609f3231..7d251e933965 100644 --- a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go @@ -14,6 +14,9 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go index f47f6ef2ca5e..53fce487a7aa 100644 --- a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go @@ -14,6 +14,9 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/data.json b/x-pack/metricbeat/module/azure/container_service/_meta/data.json index fe6623057a3a..07d8025c93da 100644 --- a/x-pack/metricbeat/module/azure/container_service/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_service/_meta/data.json @@ -1,46 +1,40 @@ { - "@timestamp" : "2020-02-28T14:47:00.000Z", - "ecs" : { - "version" : "1.4.0" - }, - "cloud" : { - "region" : "westeurope", - "provider" : "azure" + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "container_service": { + "kube_node_status_condition": { + "avg": 1 + } + }, + "dimensions": { + "status": "true" + }, + "namespace": "Microsoft.ContainerService/managedClusters", + "resource": { + "group": "obs-infrastructure", + "type": "Microsoft.ContainerService/managedClusters" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "event" : { - "dataset" : "azure.container_service", - "module" : "azure", - "duration" : 2730896000 + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerService/managedClusters/obskube", + "name": "obskube" + }, + "provider": "azure", + "region": "westeurope" }, - "metricset" : { - "name" : "container_service", - "period" : 300000 + "event": { + "dataset": "azure.container_service", + "duration": 115000, + "module": "azure" }, - "azure" : { - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerService/managedClusters", - "container_service" : { - "kube_pod_status_ready" : { - "avg" : 9 - }, - "kube_pod_status_phase" : { - "avg" : 9 - }, - "kube_node_status_allocatable_cpu_cores" : { - "avg" : 2 - }, - "kube_node_status_allocatable_memory_bytes" : { - "avg" : 2.25349632E9 - } - }, - "timegrain" : "PT5M", - "resource" : { - "type" : "Microsoft.ContainerService/managedClusters", - "group" : "obs-infrastructure", - "name" : "obskube" - } + "metricset": { + "name": "container_service", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go index baa6be30d55b..93acb39da514 100644 --- a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go @@ -14,6 +14,9 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) func TestFetchMetricset(t *testing.T) { From d5c24f47c692c21c113317948a303f60a2125773 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Mar 2020 21:47:52 +0100 Subject: [PATCH 7/7] generate from test --- .../azure/container_registry/_meta/data.json | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json index 0167132328b0..0ae5b5f3e259 100644 --- a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json @@ -1,37 +1,46 @@ { - "@timestamp" : "2020-02-28T14:47:00.000Z", - "ecs" : { - "version" : "1.4.0" - }, - "cloud" : { - "region" : "westeurope", - "provider" : "azure" + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "container_registry": { + "successful_pull_count": { + "avg": 1 + }, + "successful_push_count": { + "avg": 1 + }, + "total_pull_count": { + "avg": 1 + }, + "total_push_count": { + "avg": 1 + } + }, + "namespace": "Microsoft.ContainerRegistry/registries", + "resource": { + "group": "obs-infrastructure", + "type": "Microsoft.ContainerRegistry/registries" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "event" : { - "dataset" : "azure.container_registry", - "module" : "azure", - "duration" : 2730896000 + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerRegistry/registries/obstest", + "name": "obstest" + }, + "provider": "azure", + "region": "westeurope" }, - "metricset" : { - "name" : "container_registry", - "period" : 300000 + "event": { + "dataset": "azure.container_registry", + "duration": 115000, + "module": "azure" }, - "azure" : { - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerRegistry/registries", - "container_registry" : { - "successful_push_count" : { - "avg" : 1 - } - }, - "timegrain" : "PT5M", - "resource" : { - "type" : "Microsoft.ContainerRegistry/registries", - "group" : "obs-infrastructure", - "name" : "obstest" - } + "metricset": { + "name": "container_registry", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } }