[vSphere][datastore] Add support for additional metrics

packages/vsphere/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.0-next"
+  changes:
+    - description: Add additional metrics for datastore datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10911
 - version: "1.14.0"
   changes:
     - description: Add support for processors in datastore, host and virtualmachine.

packages/vsphere/data_stream/datastore/fields/fields.yml

@@ -6,33 +6,109 @@
       type: keyword
       # Reason to add as a dimension field: to uniquely identify the datastore
       dimension: true
-      description: Datastore name
+      description: Datastore name.
     - name: fstype
       type: keyword
       # Reason to add as a dimension field: in case datastore name is duplicated across different type.
       dimension: true
-      description: Filesystem type
+      description: Filesystem type.
+    - name: disk.capacity.bytes
+      type: long
+      metric_type: gauge
+      description: >
+        Configured size of the datastore.
+      format: bytes
+    - name: disk.capacity.usage.bytes
+      type: long
+      metric_type: gauge
+      description: >
+        The amount of storage capacity currently being consumed by datastore.
+      format: bytes
+    - name: disk.provisioned.bytes
+      type: long
+      metric_type: gauge
+      description: >
+        Amount of storage set aside for use by a datastore.
+      format: bytes
     - name: capacity.total.bytes
       type: long
       metric_type: gauge
       unit: byte
-      description: Total bytes of the datastore
+      description: Total bytes of the datastore.
       format: bytes
     - name: capacity.free.bytes
       type: long
       metric_type: gauge
       unit: byte
-      description: Free bytes of the datastore
+      description: Free bytes of the datastore.
       format: bytes
     - name: capacity.used.bytes
       type: long
       metric_type: gauge
       unit: byte
-      description: Used bytes of the datastore
+      description: Used bytes of the datastore.
       format: bytes
     - name: capacity.used.pct
       type: scaled_float
       metric_type: gauge
       unit: percent
-      description: Used percent of the datastore
+      description: Percentage of datastore capacity used.
       format: percent
+    - name: host.count
+      type: long
+      metric_type: gauge
+      description: >
+        Number of hosts associated with the datastore.
+    - name: host.names
+      type: keyword
+      description: >
+        List of all the host names associated with the datastore.
+    - name: iops
+      type: long
+      metric_type: gauge
+      description: >
+        Storage I/O Control aggregated Input/Output Operations Per Second.
+    - name: read.bytes
+      type: long
+      metric_type: gauge
+      unit: byte
+      description: >
+        Rate of reading data from the datastore.
+      format: bytes
+    - name: read.latency.total.ms
+      type: long
+      metric_type: gauge
+      unit: ms
+      description: >
+        Average amount of time for a read operation from the datastore in milliseconds.
+    - name: status
+      type: keyword
+      description: >
+        Status of the datastore.
+    - name: triggerd_alarms.*
+      type: object
+      object_type: keyword
+      description: >
+        List of all the triggerd alarms.
+    - name: vm.count
+      type: long
+      metric_type: gauge
+      description: >
+        Number of VMs associated with the datastore.
+    - name: vm.names
+      type: keyword
+      description: >
+        List of all the VM names associated with the datastore.
+    - name: write.bytes
+      type: long
+      metric_type: gauge
+      unit: byte
+      description: >
+        Rate of writing data to the datastore.
+      format: bytes
+    - name: write.latency.total.ms
+      metric_type: gauge
+      unit: ms
+      type: long
+      description: >
+        Average amount of time for a write operation from the datastore in milliseconds.

packages/vsphere/data_stream/datastore/manifest.yml

@@ -5,6 +5,19 @@ streams:
     title: vSphere datastore metrics
     description: vSphere datastore metrics
     vars:
+      - name: period
+        type: text
+        title: Period
+        description: The vSphere performance API allows for collecting host and datastore metrics at various intervals, including real-time (every 20 seconds), 5 minutes, or longer durations. Properly configuring the "period" parameter is crucial to manage usage restrictions and ensure accurate data collection. For more information on optimizing your data collection setup, refer to the [Data Collection Intervals](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-247646EA-A04B-411A-8DD4-62A3DCFCF49B.html) and [Data Collection Levels](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.monitoring.doc/GUID-25800DE4-68E5-41CC-82D9-8811E27924BC.html) documentation.
+        default: 20s
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - vsphere-datastore
       - name: processors
         type: yaml
         title: Processors

packages/vsphere/data_stream/datastore/sample_event.json

@@ -1,48 +1,50 @@
 {
-    "@timestamp": "2023-06-29T08:03:30.114Z",
+    "@timestamp": "2024-09-02T10:04:25.122Z",
     "agent": {
-        "ephemeral_id": "8b019ff3-cbda-41fa-b1ff-974d482b9694",
-        "id": "5096d7cc-1e4b-4959-abea-7355be2913a7",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "4da294a3-ad54-47f4-92c7-544e1356a0d8",
+        "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a",
+        "name": "elastic-agent-76236",
         "type": "metricbeat",
-        "version": "8.8.1"
+        "version": "8.16.0"
     },
     "data_stream": {
         "dataset": "vsphere.datastore",
-        "namespace": "ep",
+        "namespace": "86691",
         "type": "metrics"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "5096d7cc-1e4b-4959-abea-7355be2913a7",
-        "snapshot": false,
-        "version": "8.8.1"
+        "id": "b01ab3cf-51ad-4c4d-87bd-fc2d4aa59d8a",
+        "snapshot": true,
+        "version": "8.16.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "vsphere.datastore",
-        "duration": 23155458,
-        "ingested": "2023-06-29T08:03:31Z",
+        "duration": 190147614,
+        "ingested": "2024-09-02T10:04:27Z",
         "module": "vsphere"
     },
     "host": {
-        "architecture": "aarch64",
-        "containerized": false,
-        "hostname": "docker-fleet-agent",
-        "id": "d08b346fbb8f49f5a2bb1a477f8ceb54",
+        "architecture": "x86_64",
+        "containerized": true,
+        "hostname": "elastic-agent-76236",
+        "id": "e744630f9d4f43dc818e497d221bd0b2",
         "ip": [
-            "172.23.0.7"
+            "172.18.0.4",
+            "172.21.0.2"
         ],
         "mac": [
-            "02-42-AC-17-00-07"
+            "02-42-AC-12-00-04",
+            "02-42-AC-15-00-02"
         ],
-        "name": "docker-fleet-agent",
+        "name": "elastic-agent-76236",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.104-linuxkit",
+            "kernel": "5.15.153.1-microsoft-standard-WSL2",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -54,25 +56,56 @@
         "period": 10000
     },
     "service": {
-        "address": "https://elastic-package-service_vsphere-metrics_1:8989/sdk",
+        "address": "https://svc-vsphere-metrics:8989/sdk",
         "type": "vsphere"
     },
     "vsphere": {
         "datastore": {
+            "disk": {
+                "capacity": {
+                    "usage": {
+                        "bytes": 520505786368
+                    },
+                    "bytes": 1610344300544
+                },
+                "provisioned": {
+                    "bytes": 520505786368
+                }
+            },
             "capacity": {
                 "free": {
-                    "bytes": 47869427712
+                    "bytes": 37120094208
                 },
                 "total": {
-                    "bytes": 62725623808
+                    "bytes": 74686664704
                 },
                 "used": {
-                    "bytes": 14856196096,
-                    "pct": 0.237
+                    "bytes": 37566570496,
+                    "pct": 0.502988996026061
                 }
             },
             "fstype": "OTHER",
-            "name": "LocalDS_0"
+            "host": {
+                "count": 1,
+                "names": "DC0_H0"
+            },
+            "name": "LocalDS_0",
+            "read": {
+                "bytes": 1024
+            },
+            "status": "green",
+            "vm": {
+                "count": 4,
+                "names": [
+                    "DC0_C0_RP0_VM0",
+                    "DC0_C0_RP0_VM1",
+                    "DC0_H0_VM0",
+                    "DC0_H0_VM1"
+                ]
+            },
+            "write": {
+                "bytes": 450560
+            }
         }
     }
 }