feat: Replace prometheus with metricbeat by extending the elastic_stack role

ansible/roles/elastic_beats/tasks/main.yml

@@ -110,6 +110,12 @@
             hosts: ["unix:///var/run/docker.sock"]
             period: 10s
             enabled: true
+
+          - module: prometheus
+            period: 10s
+            hosts: ["{{ private_ip }}:{{ prometheus_port }}"]
+            metrics_path: "/metrics"
+            namespace: "drive_tenderdash"
     output_conf:
       elasticsearch:
         hosts:

ansible/roles/elastic_beats/vars/tenderdash.yml

@@ -22,3 +22,10 @@ platform_filebeat_inputs:
           fields:
             - from: "json.level"
               to: "log.level"
+
+metricbeat_modules:
+  - module: prometheus
+    period: 10s
+    hosts: ["{{ private_ip }}:{{ prometheus_port }}"]
+    metrics_path: "/metrics"
+    namespace: "drive_tenderdash"

ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json

@@ -0,0 +1,11 @@
+{
+    "description": "Parse round number from Tenderdash logs",
+    "processors": [
+      {
+        "grok": {
+          "field": "message",
+          "patterns": ["Processed proposal with .* for height: %{NUMBER:height}, round: %{NUMBER:round}"]
+        }
+      }
+    ]
+  }

ansible/roles/elastic_stack/files/watches/error_logs_watch.json

@@ -0,0 +1,33 @@
+{
+    "trigger": {
+      "schedule": {
+        "interval": "10s"
+      }
+    },
+    "input": {
+      "search": {
+        "request": {
+          "indices": ["logs-*"],
+          "body": {
+            "query": {
+              "bool": {
+                "should": [
+                  { "match": { "log.level": "ERROR" }},
+                  { "match": { "log.level": "FATAL" }}
+                ],
+                "minimum_should_match": 1
+              }
+            }
+          }
+        }
+      }
+    },
+    "actions": {
+      "log_alert": {
+        "logging": {
+          "text": "Alert triggered: {{ctx.payload}}"
+        }
+      }
+    }
+  }
+

ansible/roles/elastic_stack/files/watches/high_round_watch.json

@@ -0,0 +1,41 @@
+{
+    "trigger": {
+      "schedule": {
+        "interval": "10s"
+      }
+    },
+    "input": {
+      "search": {
+        "request": {
+          "indices": ["logs-*"],
+          "body": {
+            "query": {
+              "bool": {
+                "must": [
+                  {
+                    "match": {
+                      "message": "Processed proposal"
+                    }
+                  },
+                  {
+                    "range": {
+                      "round": {
+                        "gt": 5
+                      }
+                    }
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    },
+    "actions": {
+      "log_alert": {
+        "logging": {
+          "text": "Alert triggered: {{ctx.payload}}"
+        }
+      }
+    }
+  }

ansible/roles/elastic_stack/files/watches/no_logs_watch.json

@@ -0,0 +1,46 @@
+{
+    "trigger": {
+      "schedule": {
+        "interval": "5m"
+      }
+    },
+    "input": {
+      "search": {
+        "request": {
+          "indices": ["logs-*"],
+          "body": {
+            "query": {
+              "bool": {
+                "must_not": {
+                  "exists": {
+                    "field": "message"
+                  }
+                },
+                "filter": {
+                  "range": {
+                    "@timestamp": {
+                      "gte": "now-5m"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "condition": {
+      "compare": {
+        "ctx.payload.hits.total": {
+          "eq": 0
+        }
+      }
+    },
+    "actions": {
+      "log_alert": {
+        "logging": {
+          "text": "Alert triggered: {{ctx.payload}}"
+        }
+      }
+    }
+  }

ansible/roles/elastic_stack/tasks/main.yml

@@ -119,3 +119,30 @@
   ansible.builtin.import_tasks: configure_cluster.yml
   run_once: true
   delegate_to: '{{ play_hosts | first }}'
+
+
+- name: Upload and apply Tenderdash log parsing pipeline
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_url }}/logstash/_pipeline/process_proposal_pipeline"
+    method: PUT
+    body: "{{ lookup('file', 'files/pipelines/process_proposal_pipeline.json') }}"
+    body_format: json
+    user: "{{ elastic_username }}"
+    password: "{{ elastic_password }}"
+  when: inventory_hostname in groups['elasticsearch']
+
+- name: Create Elasticsearch Watches
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_url }}/_watcher/watch/{{ item.name }}"
+    method: PUT
+    body: "{{ lookup('file', item.file) }}"
+    body_format: json
+    user: "{{ elastic_username }}"
+    password: "{{ elastic_password }}"
+  loop:
+    - { name: 'error_logs_watch', file: 'files/watches/error_logs_watch.json' }
+    - { name: 'high_round_watch', file: 'files/watches/high_round_watch.json' }
+    - { name: 'no_logs_watch', file: 'files/watches/no_logs_watch.json' }
+  loop_control:
+    label: "{{ item.name }}"
+  when: inventory_hostname in groups['elasticsearch']

ansible/roles/elastic_stack/vars/main.yml

@@ -4,3 +4,4 @@ bundle_path: '{{ elastic_path }}/bundle'
 certs_path: '{{ elastic_path }}/certificates'
 data_path: '{{ elastic_path }}/data'
 kibana_port: 5601
+elasticsearch_url: "http://{{ hostvars[inventory_hostname].private_ip }}:9200"