diff --git a/packages/vsphere/changelog.yml b/packages/vsphere/changelog.yml index 6ed2b4677c7..14815c349e9 100644 --- a/packages/vsphere/changelog.yml +++ b/packages/vsphere/changelog.yml @@ -1,6 +1,9 @@ # newer versions go on top - version: "1.15.0-next" changes: + - description: Add additional metrics for datastore datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/10911 - description: Add additional metrics for host datastream. type: enhancement link: https://github.com/elastic/integrations/pull/10894 diff --git a/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json new file mode 100644 index 00000000000..b44197b3212 --- /dev/null +++ b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json @@ -0,0 +1,55 @@ +{ + "events": [ + { + "vsphere": { + "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, + "capacity": { + "free": { + "bytes": 37120094208 + }, + "total": { + "bytes": 74686664704 + }, + "used": { + "bytes": 37566570496, + "pct": 0.502988996026061 + } + }, + "fstype": "OTHER", + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } + } + } + } + ] +} \ No newline at end of file diff --git a/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json new file mode 100644 index 00000000000..6e1ac127741 --- /dev/null +++ b/packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json @@ -0,0 +1,58 @@ +{ + "expected": [ + { + "ecs": { + "version": "8.11.0" + }, + "vsphere": { + "datastore": { + "capacity": { + "free": { + "bytes": 37120094208 + }, + "total": { + "bytes": 74686664704 + }, + "used": { + "bytes": 37566570496, + "pct": 0.502988996026061 + } + }, + "disk": { + "capacity": { + "bytes": 1610344300544, + "usage": { + "bytes": 520505786368 + } + }, + "provisioned": { + "bytes": 520505786368 + } + }, + "fstype": "OTHER", + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } + } + } + } + ] +} \ No newline at end of file diff --git a/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs b/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs index 322880bb3bb..fdda4f6f2a3 100644 --- a/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs +++ b/packages/vsphere/data_stream/datastore/agent/stream/stream.yml.hbs @@ -8,7 +8,13 @@ username: {{username}} password: {{password}} # If insecure is true, don't verify the server's certificate chain insecure: {{insecure}} + +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} + {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/vsphere/data_stream/datastore/fields/fields.yml b/packages/vsphere/data_stream/datastore/fields/fields.yml index 5a7f51316d1..aff2e039924 100644 --- a/packages/vsphere/data_stream/datastore/fields/fields.yml +++ b/packages/vsphere/data_stream/datastore/fields/fields.yml @@ -63,11 +63,6 @@ type: keyword description: > List of all the host names associated with the datastore. - - name: iops - type: long - metric_type: gauge - description: > - Storage I/O Control aggregated Input/Output Operations Per Second. - name: read.bytes type: long metric_type: gauge @@ -75,21 +70,10 @@ description: > Rate of reading data from the datastore. format: bytes - - name: read.latency.total.ms - type: long - metric_type: gauge - unit: ms - description: > - Average amount of time for a read operation from the datastore in milliseconds. - name: status type: keyword description: > Status of the datastore. - - name: triggerd_alarms.* - type: object - object_type: keyword - description: > - List of all the triggerd alarms. - name: vm.count type: long metric_type: gauge @@ -106,9 +90,26 @@ description: > Rate of writing data to the datastore. format: bytes - - name: write.latency.total.ms - metric_type: gauge - unit: ms - type: long + - name: triggered_alarms + type: group description: > - Average amount of time for a write operation from the datastore in milliseconds. \ No newline at end of file + List of all the triggered alarms. + fields: + - name: status + type: keyword + description: Status of the alarm. + - name: triggered_time + type: date + description: Time when the alarm was triggered. + - name: description + type: keyword + description: Description of the alarm. + - name: entity_name + type: keyword + description: Name of the entity associated with the alarm. + - name: name + type: keyword + description: Name of the alarm. + - name: id + type: keyword + description: Unique identifier for the alarm. diff --git a/packages/vsphere/data_stream/datastore/manifest.yml b/packages/vsphere/data_stream/datastore/manifest.yml index 48dad09edc7..e2eedfa706c 100644 --- a/packages/vsphere/data_stream/datastore/manifest.yml +++ b/packages/vsphere/data_stream/datastore/manifest.yml @@ -5,6 +5,19 @@ streams: title: vSphere datastore metrics description: vSphere datastore metrics vars: + - name: period + type: text + title: Period + description: The vSphere performance API allows for collecting host and datastore metrics at various intervals, including real-time (every 20 seconds), 5 minutes, or longer durations. Properly configuring the "period" parameter is crucial to manage usage restrictions and ensure accurate data collection. For more information on optimizing your data collection setup, refer to the [Data Collection Intervals](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html) and [Data Collection Levels](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html) documentation. + default: 20s + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - vsphere-datastore - name: processors type: yaml title: Processors diff --git a/packages/vsphere/data_stream/datastore/sample_event.json b/packages/vsphere/data_stream/datastore/sample_event.json index ed9113d9b8c..f48886186e3 100644 --- a/packages/vsphere/data_stream/datastore/sample_event.json +++ b/packages/vsphere/data_stream/datastore/sample_event.json @@ -1,48 +1,50 @@ { - "@timestamp": "2023-06-29T08:03:30.114Z", + "@timestamp": "2024-09-02T10:04:25.122Z", "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", + "ephemeral_id": "4da294a3-ad54-47f4-92c7-544e1356a0d8", + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "name": "elastic-agent-76236", "type": "metricbeat", - "version": "8.8.1" + "version": "8.16.0" }, "data_stream": { "dataset": "vsphere.datastore", - "namespace": "ep", + "namespace": "86691", "type": "metrics" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "snapshot": true, + "version": "8.16.0" }, "event": { "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 190147614, + "ingested": "2024-09-02T10:04:27Z", "module": "vsphere" }, "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "architecture": "x86_64", + "containerized": true, + "hostname": "elastic-agent-76236", + "id": "e744630f9d4f43dc818e497d221bd0b2", "ip": [ - "172.23.0.7" + "172.18.0.4", + "172.21.0.2" ], "mac": [ - "02-42-AC-17-00-07" + "02-42-AC-12-00-04", + "02-42-AC-15-00-02" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-76236", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.104-linuxkit", + "kernel": "5.15.153.1-microsoft-standard-WSL2", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", @@ -54,25 +56,56 @@ "period": 10000 }, "service": { - "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk", + "address": "https://svc-vsphere-metrics:8989/sdk", "type": "vsphere" }, "vsphere": { "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, "capacity": { "free": { - "bytes": 47869427712 + "bytes": 37120094208 }, "total": { - "bytes": 62725623808 + "bytes": 74686664704 }, "used": { - "bytes": 14856196096, - "pct": 0.237 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "OTHER", - "name": "LocalDS_0" + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } } } } \ No newline at end of file diff --git a/packages/vsphere/docs/README.md b/packages/vsphere/docs/README.md index 5ea5bf7633d..36dcaa727ff 100644 --- a/packages/vsphere/docs/README.md +++ b/packages/vsphere/docs/README.md @@ -172,50 +172,52 @@ An example event for `datastore` looks as following: ```json { - "@timestamp": "2023-06-29T08:03:30.114Z", + "@timestamp": "2024-09-02T10:04:25.122Z", "agent": { - "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694", - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "name": "docker-fleet-agent", + "ephemeral_id": "4da294a3-ad54-47f4-92c7-544e1356a0d8", + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "name": "elastic-agent-76236", "type": "metricbeat", - "version": "8.8.1" + "version": "8.16.0" }, "data_stream": { "dataset": "vsphere.datastore", - "namespace": "ep", + "namespace": "86691", "type": "metrics" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "5096d7cc-1e4b-4959-abea-7355be2913a7", - "snapshot": false, - "version": "8.8.1" + "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a", + "snapshot": true, + "version": "8.16.0" }, "event": { "agent_id_status": "verified", "dataset": "vsphere.datastore", - "duration": 23155458, - "ingested": "2023-06-29T08:03:31Z", + "duration": 190147614, + "ingested": "2024-09-02T10:04:27Z", "module": "vsphere" }, "host": { - "architecture": "aarch64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "architecture": "x86_64", + "containerized": true, + "hostname": "elastic-agent-76236", + "id": "e744630f9d4f43dc818e497d221bd0b2", "ip": [ - "172.23.0.7" + "172.18.0.4", + "172.21.0.2" ], "mac": [ - "02-42-AC-17-00-07" + "02-42-AC-12-00-04", + "02-42-AC-15-00-02" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-76236", "os": { "codename": "focal", "family": "debian", - "kernel": "5.10.104-linuxkit", + "kernel": "5.15.153.1-microsoft-standard-WSL2", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", @@ -227,25 +229,56 @@ An example event for `datastore` looks as following: "period": 10000 }, "service": { - "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk", + "address": "https://svc-vsphere-metrics:8989/sdk", "type": "vsphere" }, "vsphere": { "datastore": { + "disk": { + "capacity": { + "usage": { + "bytes": 520505786368 + }, + "bytes": 1610344300544 + }, + "provisioned": { + "bytes": 520505786368 + } + }, "capacity": { "free": { - "bytes": 47869427712 + "bytes": 37120094208 }, "total": { - "bytes": 62725623808 + "bytes": 74686664704 }, "used": { - "bytes": 14856196096, - "pct": 0.237 + "bytes": 37566570496, + "pct": 0.502988996026061 } }, "fstype": "OTHER", - "name": "LocalDS_0" + "host": { + "count": 1, + "names": "DC0_H0" + }, + "name": "LocalDS_0", + "read": { + "bytes": 1024 + }, + "status": "green", + "vm": { + "count": 4, + "names": [ + "DC0_C0_RP0_VM0", + "DC0_C0_RP0_VM1", + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "write": { + "bytes": 450560 + } } } } @@ -287,16 +320,18 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | vsphere.datastore.fstype | Filesystem type. | keyword | | | | vsphere.datastore.host.count | Number of hosts associated with the datastore. | long | | gauge | | vsphere.datastore.host.names | List of all the host names associated with the datastore. | keyword | | | -| vsphere.datastore.iops | Storage I/O Control aggregated Input/Output Operations Per Second. | long | | gauge | | vsphere.datastore.name | Datastore name. | keyword | | | | vsphere.datastore.read.bytes | Rate of reading data from the datastore. | long | byte | gauge | -| vsphere.datastore.read.latency.total.ms | Average amount of time for a read operation from the datastore in milliseconds. | long | ms | gauge | | vsphere.datastore.status | Status of the datastore. | keyword | | | -| vsphere.datastore.triggerd_alarms.\* | List of all the triggerd alarms. | object | | | +| vsphere.datastore.triggered_alarms.description | Description of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.entity_name | Name of the entity associated with the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.id | Unique identifier for the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.name | Name of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.status | Status of the alarm. | keyword | | | +| vsphere.datastore.triggered_alarms.triggered_time | Time when the alarm was triggered. | date | | | | vsphere.datastore.vm.count | Number of VMs associated with the datastore. | long | | gauge | | vsphere.datastore.vm.names | List of all the VM names associated with the datastore. | keyword | | | | vsphere.datastore.write.bytes | Rate of writing data to the datastore. | long | byte | gauge | -| vsphere.datastore.write.latency.total.ms | Average amount of time for a write operation from the datastore in milliseconds. | long | ms | gauge | ### Host