Merge pull request #12734 from Security-Onion-Solutions/metrics

Add Elastic Agent Status Metrics

salt/common/tools/sbin/so-common

@@ -248,6 +248,11 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+get_agent_count() {
+
+	AGENTCOUNT=$(/usr/sbin/so-elasticagent-status | grep -wF active | awk '{print $2}')
+}
+
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -570,8 +575,9 @@ sync_options() {
 	set_version
 	set_os
 	salt_minion_count
+	get_agent_count
 
-	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)"
+	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT:$AGENTCOUNT/$(read_feat)"
 }
 
 systemctl_func() {

salt/manager/init.sls

@@ -27,6 +27,15 @@ repo_log_dir:
       - user
       - group
 
+agents_log_dir:
+  file.directory:
+    - name: /opt/so/log/agents
+    - user: root
+    - group: root
+    - recurse:
+      - user
+      - group
+
 yara_log_dir:
   file.directory:
     - name: /opt/so/log/yarasync
@@ -101,6 +110,17 @@ so-repo-sync:
     - hour: '{{ MANAGERMERGED.reposync.hour }}'
     - minute: '{{ MANAGERMERGED.reposync.minute }}'
 
+so_fleetagent_status:
+  cron.present:
+    - name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1
+    - identifier: so_fleetagent_status
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 socore_own_saltstack:
   file.directory:
     - name: /opt/so/saltstack

salt/manager/tools/sbin/so-elasticagent-status

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+curl -s -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agent_status" | jq .

salt/telegraf/defaults.yaml

@@ -11,6 +11,7 @@ telegraf:
     quiet: 'false'
   scripts:
     eval:
+      - agentstatus.sh
       - checkfiles.sh
       - influxdbsize.sh
       - lasthighstate.sh
@@ -23,6 +24,7 @@ telegraf:
       - zeekcaptureloss.sh
       - zeekloss.sh
     standalone:
+      - agentstatus.sh
       - checkfiles.sh
       - eps.sh
       - influxdbsize.sh
@@ -38,6 +40,7 @@ telegraf:
       - zeekloss.sh
       - features.sh
     manager:
+      - agentstatus.sh
       - influxdbsize.sh
       - lasthighstate.sh
       - os.sh
@@ -46,6 +49,7 @@ telegraf:
       - sostatus.sh
       - features.sh
     managersearch:
+      - agentstatus.sh
       - eps.sh
       - influxdbsize.sh
       - lasthighstate.sh

salt/telegraf/enabled.sls

@@ -56,6 +56,7 @@ so-telegraf:
       - /opt/so/log/raid:/var/log/raid:ro
       - /opt/so/log/sostatus:/var/log/sostatus:ro
       - /opt/so/log/salt:/var/log/salt:ro
+      - /opt/so/log/agents:/var/log/agents:ro
       {% if DOCKER.containers['so-telegraf'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-telegraf'].custom_bind_mounts %}
       - {{ BIND }}

salt/telegraf/scripts/agentstatus.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    LOGFILE=/var/log/agents/agentstatus.log
+
+    # Check to see if the file is there yet so we don't break install verification since there is a 5 minute delay for this file to show up
+    if [ -f $LOGFILE ]; then
+        ONLINE=$(cat $LOGFILE | grep -wF online | awk '{print $2}' | tr -d ',')
+        ERROR=$(cat $LOGFILE | grep -wF error | awk '{print $2}' | tr -d ',')
+        INACTIVE=$(cat $LOGFILE | grep -wF inactive | awk '{print $2}' | tr -d ',')
+        OFFLINE=$(cat $LOGFILE | grep -wF offline | awk '{print $2}' | tr -d ',')
+        UPDATING=$(cat $LOGFILE | grep -wF updating | awk '{print $2}' | tr -d ',')
+        UNENROLLED=$(cat $LOGFILE | grep -wF unenrolled | awk '{print $2}' | tr -d ',')
+        OTHER=$(cat $LOGFILE | grep -wF other | awk '{print $2}' | tr -d ',')
+        EVENTS=$(cat $LOGFILE | grep -wF events | awk '{print $2}' | tr -d ',')
+        TOTAL=$(cat $LOGFILE | grep -wF total | awk '{print $2}' | tr -d ',')
+        ALL=$(cat $LOGFILE | grep -wF all | awk '{print $2}' | tr -d ',')
+        ACTIVE=$(cat $LOGFILE | grep -wF active | awk '{print $2}')
+
+        echo "agentstatus online=$ONLINE,error=$ERROR,inactive=$INACTIVE,offline=$OFFLINE,updating=$UPDATING,unenrolled=$UNENROLLED,other=$OTHER,events=$EVENTS,total=$TOTAL,all=$ALL,active=$ACTIVE"
+    fi
+
+fi
+
+exit 0