across: Implement metrics API (kadalu#643)

Updates: kadalu#8 
Signed-off-by: Shree Vatsa N <[email protected]>

Author: vatsa287

csi/controllerserver.py

@@ -517,7 +517,7 @@ def ListVolumes(self, request, context):
                 "volume_id": value.get("name"),
                 "capacity_bytes": value.get("size"),
             }
-        } for value in pvcs]
+        } for value in pvcs if value is not None]
 
         return csi_pb2.ListVolumesResponse(entries=entries,
                                            next_token=next_token)

csi/exporter.py

@@ -1,128 +1,140 @@
 import os
-import pathlib
-import time
 import logging
-
-from prometheus_client.core import GaugeMetricFamily, \
-     CounterMetricFamily, REGISTRY
-from prometheus_client import start_http_server
-
-from volumeutils import (HOSTVOL_MOUNTDIR, PV_TYPE_SUBVOL,
-                         yield_pvc_from_mntdir)
+import uvicorn
+from fastapi import FastAPI
+from volumeutils import HOSTVOL_MOUNTDIR, yield_pvc_from_mntdir
 from kadalulib import logging_setup, logf
 
-
-class CsiMetricsCollector(object):
-    def collect(self):
-        # TODO: Add more labels
-        capacity_labels = ['storage_name']
-        capacity_bytes = GaugeMetricFamily(
-            'kadalu_storage_capacity_bytes',
-            'Kadalu Storage Capacity',
-            labels=capacity_labels
-        )
-        capacity_used_bytes = GaugeMetricFamily(
-            'kadalu_storage_capacity_used_bytes',
-            'Kadalu Storage Used Capacity',
-            labels=capacity_labels
-        )
-        capacity_free_bytes = GaugeMetricFamily(
-            'kadalu_storage_capacity_free_bytes',
-            'Kadalu Storage Free Capacity',
-            labels=capacity_labels
-        )
-        inodes_count = CounterMetricFamily(
-            'kadalu_storage_inodes_count',
-            'Kadalu Storage Inodes Count',
-            labels=capacity_labels
-        )
-        inodes_used_count = CounterMetricFamily(
-            'kadalu_storage_inodes_used_count',
-            'Kadalu Storage Inodes used Count',
-            labels=capacity_labels
-        )
-        inodes_free_count = CounterMetricFamily(
-            'kadalu_storage_inodes_free_count',
-            'Kadalu Storage Inodes free Count',
-            labels=capacity_labels
-        )
-        pv_capacity_bytes = GaugeMetricFamily(
-            'kadalu_storage_pv_capacity_bytes',
-            'Kadalu Storage PV Capacity',
-            labels=capacity_labels+["pv"]
-        )
-        pv_capacity_used_bytes = GaugeMetricFamily(
-            'kadalu_storage_pv_capacity_used_bytes',
-            'Kadalu Storage PV Used Capacity',
-            labels=capacity_labels+["pv"]
-        )
-        pv_capacity_free_bytes = GaugeMetricFamily(
-            'kadalu_storage_pv_capacity_free_bytes',
-            'Kadalu Storage PV Free Capacity',
-            labels=capacity_labels+["pv"]
-        )
-
-        for dirname in os.listdir(HOSTVOL_MOUNTDIR):
-            labels = [dirname]  # TODO: Add more labels
-            pth = os.path.join(HOSTVOL_MOUNTDIR, dirname)
-            if os.path.ismount(pth):
-                stat = os.statvfs(pth)
-
-                # Capacity
-                total = stat.f_bsize * stat.f_blocks
-                free = stat.f_bsize * stat.f_bavail
-                used = total - free
-                capacity_bytes.add_metric(labels, total)
-                capacity_free_bytes.add_metric(labels, free)
-                capacity_used_bytes.add_metric(labels, used)
-
-                # Inodes
-                total = stat.f_files
-                free = stat.f_favail
-                used = total - free
-                inodes_count.add_metric(labels, total)
-                inodes_free_count.add_metric(labels, free)
-                inodes_used_count.add_metric(labels, used)
-
-                # Gathers capacity metrics for each subvol
-                for pvc in yield_pvc_from_mntdir(os.path.join(pth, "info")):
-                    if pvc is None:
-                        continue
-                    pvcpath_full = os.path.join(pth, pvc.get("path_prefix"),
-                                           pvc.get("name"))
-                    pvclabels = labels + [pvc.get("name")]
-
-                    stat = os.statvfs(pvcpath_full)
-
-                    # Capacity
-                    total = stat.f_bsize * stat.f_blocks
-                    free = stat.f_bsize * stat.f_bavail
-                    used = total - free
-                    pv_capacity_bytes.add_metric(pvclabels, total)
-                    pv_capacity_free_bytes.add_metric(pvclabels, free)
-                    pv_capacity_used_bytes.add_metric(pvclabels, used)
-
-        yield capacity_bytes
-        yield capacity_free_bytes
-        yield capacity_used_bytes
-        yield inodes_count
-        yield inodes_free_count
-        yield inodes_used_count
-        yield pv_capacity_bytes
-        yield pv_capacity_used_bytes
-        yield pv_capacity_free_bytes
-
-
-REGISTRY.register(CsiMetricsCollector())
+app = FastAPI()
+
+@app.get("/_api/metrics")
+def metrics():
+    """
+    Gathers storage and pvcs metrics.
+    Starts process by exposing the data collected in port 8050 at '/_api/metrics'.
+    """
+
+    data = {
+        "pod": {},
+        "storages": []
+    }
+
+    memory_usage_in_bytes = 0
+    cpu_usage_in_nanoseconds = 0
+
+    memory_usage_file_path = '/sys/fs/cgroup/memory/memory.usage_in_bytes'
+    with open(memory_usage_file_path, 'r') as memory_fd:
+        memory_usage_in_bytes = int(memory_fd.read().strip())
+
+    cpu_usage_file_path = '/sys/fs/cgroup/cpu/cpuacct.usage'
+    with open(cpu_usage_file_path, 'r') as cpu_fd:
+        cpu_usage_in_nanoseconds = int(cpu_fd.read().strip())
+
+    data["pod"] = {
+        "memory_usage_in_bytes": memory_usage_in_bytes,
+        "cpu_usage_in_nanoseconds": cpu_usage_in_nanoseconds
+    }
+
+    if os.environ.get("CSI_ROLE", "-") == "nodeplugin":
+        pod_name_path = '/etc/hostname'
+        with open(pod_name_path, 'r') as pod_fd:
+            pod_name = pod_fd.read().strip()
+            data["pod"].update({"pod_name": pod_name})
+
+    # Handle condition for no storage & PVC,
+    # sometimes storage name is not shown at /mnt until server is mounted.
+    if len(os.listdir(HOSTVOL_MOUNTDIR)) == 0:
+        logging.debug(logf(
+            "No storage-pool found! Try again by creating a storage.",
+            HOSTVOL_MOUNTDIR=HOSTVOL_MOUNTDIR
+        ))
+        return data
+
+    # Gathers metrics for each storage
+    for dirname in os.listdir(HOSTVOL_MOUNTDIR):
+        storage_path = os.path.join(HOSTVOL_MOUNTDIR, dirname)
+
+        if os.path.ismount(storage_path):
+
+            stat = os.statvfs(storage_path)
+
+            # Storage Capacity
+            total_capacity_bytes = stat.f_bsize * stat.f_blocks
+            free_capacity_bytes = stat.f_bsize * stat.f_bavail
+            used_capacity_bytes = total_capacity_bytes - free_capacity_bytes
+
+            # Storage Inodes
+            total_inodes = stat.f_files
+            free_inodes = stat.f_favail
+            used_inodes = total_inodes - free_inodes
+
+            storage = {
+                "name": dirname,
+                "total_capacity_bytes": total_capacity_bytes,
+                "free_capacity_bytes": free_capacity_bytes,
+                "used_capacity_bytes": used_capacity_bytes,
+                "total_inodes": total_inodes,
+                "free_inodes": free_inodes,
+                "used_inodes": used_inodes,
+                "pvc": []
+            }
+
+            storage_info_path = os.path.join(storage_path, "info")
+            if not os.path.exists(storage_info_path):
+                data["storages"].append(storage)
+                logging.warning(logf(
+                    "No PVC found. Sending only storage metrics"
+                ))
+                return data
+
+            # Gathers metrics for each subvol[PVC]
+            for pvc in yield_pvc_from_mntdir(storage_info_path):
+
+                # Handle condition when PVC is created and then deleted,
+                # Leaving an empty leaf directory with path prefix.
+                if pvc is None:
+                    logging.warning(logf(
+                        "PVC JSON file not found. PVC must have been deleted. Trying again!"
+                    ))
+                    # Skip loop for now and look for any new possible healthy PVC
+                    continue
+
+                pvcname = pvc.get("name")
+                pvcpath = os.path.join(storage_path, pvc.get("path_prefix"), pvcname)
+
+                stat = os.statvfs(pvcpath)
+
+                # PVC Capacity
+                total_pvc_capacity_bytes = stat.f_bsize * stat.f_blocks
+                free_pvc_capacity_bytes = stat.f_bsize * stat.f_bavail
+                used_pvc_capacity_bytes = total_pvc_capacity_bytes - free_pvc_capacity_bytes
+
+                # PVC Inodes
+                total_pvc_inodes = stat.f_files
+                free_pvc_inodes = stat.f_favail
+                used_pvc_inodes = total_pvc_inodes - free_pvc_inodes
+
+                pvc = {
+                    "pvc_name": pvcname,
+                    "total_pvc_capacity_bytes": total_pvc_capacity_bytes,
+                    "free_pvc_capacity_bytes": free_pvc_capacity_bytes,
+                    "used_pvc_capacity_bytes": used_pvc_capacity_bytes,
+                    "total_pvc_inodes": total_pvc_inodes,
+                    "free_pvc_inodes": free_pvc_inodes,
+                    "used_pvc_inodes": used_pvc_inodes
+                }
+
+                storage["pvc"].append(pvc)
+            data["storages"].append(storage)
+
+    return data
 
 
 if __name__ == "__main__":
-    logging_setup()
 
-    start_http_server(8000)
-    logging.info(
-        logf("Started Kadalu Storage CSI Metrics exporter.", port=8000)
-    )
+    logging_setup()
+    logging.info(logf(
+        "Started metrics exporter process at port 8050"
+    ))
 
-    while True:
-        time.sleep(5)
+    uvicorn.run("exporter:app", host="0.0.0.0", port=8050, log_level="info")

csi/volumeutils.py

@@ -908,6 +908,7 @@ def generate_client_volfile(volname):
     Template(content).stream(**data).dump(client_volfile)
     return True
 
+
 def send_signal_to_process(volname, out, sig):
     """Sends the signal to one of the process"""
 
@@ -1149,6 +1150,7 @@ def mount_glusterfs_with_host(volname, mountpoint, hosts, options=None, is_clien
 
     return
 
+
 def check_external_volume(pv_request, host_volumes):
     """Mount hosting volume"""
     # Assumption is, this has to have 'hostvol_type' as External.
@@ -1242,6 +1244,7 @@ def yield_pvc_from_mntdir(mntdir):
         yield None
     for child in os.listdir(mntdir):
         name = os.path.join(mntdir, child)
+
         if os.path.isdir(name) and len(os.listdir(name)):
             yield from yield_pvc_from_mntdir(name)
         elif name.endswith('json'):
@@ -1290,6 +1293,7 @@ def wrap_pvc(pvc_gen):
     except StopIteration as errmsg:
         return errmsg
 
+
 def yield_list_of_pvcs(max_entries=0):
     """Yields list of PVCs limited at 'max_entries'"""
     # List of dicts containing data of PVC from info_file (with extra keys,

helm/kadalu/charts/operator/templates/deployment.yaml

@@ -16,6 +16,9 @@ spec:
     metadata:
       labels:
         name: kadalu
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8050"
     spec:
       serviceAccountName: kadalu-operator
       containers:

manifests/kadalu-operator-microk8s.yaml

@@ -282,6 +282,9 @@ spec:
     metadata:
       labels:
         name: kadalu
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8050"
     spec:
       serviceAccountName: kadalu-operator
       containers:

manifests/kadalu-operator-openshift.yaml

@@ -313,6 +313,9 @@ spec:
     metadata:
       labels:
         name: kadalu
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8050"
     spec:
       serviceAccountName: kadalu-operator
       containers:

manifests/kadalu-operator-rke.yaml

@@ -282,6 +282,9 @@ spec:
     metadata:
       labels:
         name: kadalu
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8050"
     spec:
       serviceAccountName: kadalu-operator
       containers:

manifests/kadalu-operator.yaml

@@ -282,6 +282,9 @@ spec:
     metadata:
       labels:
         name: kadalu
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8050"
     spec:
       serviceAccountName: kadalu-operator
       containers:

operator/Dockerfile

@@ -43,6 +43,8 @@ COPY lib/kadalulib.py                       /kadalu/kadalulib.py
 COPY cli/kubectl_kadalu/utils.py            /kadalu/utils.py
 COPY operator/main.py                       /kadalu/
 COPY operator/start.py                      /kadalu/
+COPY operator/metrics.py                    /kadalu/
+COPY operator/exporter.py                   /kadalu/
 COPY cli/build/kubectl-kadalu               /usr/bin/kubectl-kadalu
 COPY lib/startup.sh                         /kadalu/startup.sh