Merge pull request #13607 from Security-Onion-Solutions/esver

use Elasticsearch version for some containers

salt/common/tools/sbin/so-common

@@ -8,12 +8,6 @@
 # Elastic agent is not managed by salt. Because of this we must store this base information in a 
 # script that accompanies the soup system. Since so-common is one of those special soup files, 
 # and since this same logic is required during installation, it's included in this file.
-ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
-ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
 
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
@@ -174,6 +168,46 @@ check_salt_minion_status() {
 	return $status
 }
 
+# Compare es versions and return the highest version
+compare_es_versions() {
+    # Save the original IFS
+    local OLD_IFS="$IFS"
+
+    IFS=.
+    local i ver1=($1) ver2=($2)
+
+    # Restore the original IFS
+    IFS="$OLD_IFS"
+
+    # Determine the maximum length between the two version arrays
+    local max_len=${#ver1[@]}
+    if [[ ${#ver2[@]} -gt $max_len ]]; then
+        max_len=${#ver2[@]}
+    fi
+
+    # Compare each segment of the versions
+    for ((i=0; i<max_len; i++)); do
+		# If a segment in ver1 or ver2 is missing, set it to 0
+        if [[ -z ${ver1[i]} ]]; then
+            ver1[i]=0
+        fi
+        if [[ -z ${ver2[i]} ]]; then
+            ver2[i]=0
+        fi
+        if ((10#${ver1[i]} > 10#${ver2[i]})); then
+            echo "$1"
+            return 0
+        fi
+        if ((10#${ver1[i]} < 10#${ver2[i]})); then
+            echo "$2"
+            return 0
+        fi
+    done
+
+    echo "$1"  # If versions are equal, return either
+    return 0
+}
+
 copy_new_files() {
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
@@ -263,11 +297,6 @@ fail() {
 	exit 1
 }
 
-get_random_value() {
-	length=${1:-20}
-	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
-}
-
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
 		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
@@ -276,6 +305,27 @@ get_agent_count() {
 	fi
 }
 
+get_elastic_agent_vars() {
+	local path="${1:-/opt/so/saltstack/default}"
+	local defaultsfile="${path}/salt/elasticsearch/defaults.yaml"
+
+	if [ -f "$defaultsfile" ]; then
+		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+	else
+		fail "Could not find salt/elasticsearch/defaults.yaml"
+	fi
+}
+
+get_random_value() {
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -627,6 +677,7 @@ has_uppercase() {
 }
 
 update_elastic_agent() {
+	get_elastic_agent_vars
 	echo "Checking if Elastic Agent update is necessary..."
 	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
 }

salt/common/tools/sbin/so-image-common

@@ -112,6 +112,10 @@ update_docker_containers() {
     container_list
   fi
 
+  # all the images using ELASTICSEARCHDEFAULTS.elasticsearch.version
+  # does not include so-elastic-fleet since that container uses so-elastic-agent image
+  local IMAGES_USING_ES_VERSION=("so-elastic-fleet-package-registry" "so-elastic-agent" "so-kibana" "so-logstash" "so-elasticsearch")
+
   rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
   mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
 
@@ -139,8 +143,24 @@ update_docker_containers() {
       $PROGRESS_CALLBACK $i
     fi
 
+    # use version defined in elasticsearch defaults.yaml if an es container
+    if [[ " ${IMAGES_USING_ES_VERSION[*]} " =~ [[:space:]]${i}[[:space:]] ]]; then
+      local UPDATE_DIR='/tmp/sogh/securityonion'
+      local v1=0
+      local v2=0
+      if [[ -f "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" ]]; then
+        v1=$(egrep " +version: " "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
+      fi
+      if [[ -f "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" ]]; then
+        v2=$(egrep " +version: " "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
+      fi
+      local highest_es_version=$(compare_es_versions "$v1" "$v2")
+      local image=$i:$highest_es_version$IMAGE_TAG_SUFFIX
+    # use the so version for the version
+    else
+      local image=$i:$VERSION$IMAGE_TAG_SUFFIX
+    fi
     # Pull down the trusted docker image
-    local image=$i:$VERSION$IMAGE_TAG_SUFFIX
     run_check_net_err \
     "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
     "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 

salt/elastic-fleet-package-registry/enabled.sls

@@ -7,14 +7,15 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 include:
   - elastic-fleet-package-registry.config
   - elastic-fleet-package-registry.sostatus
 
 so-elastic-fleet-package-registry:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-fleet-package-registry:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-fleet-package-registry:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
     - name: so-elastic-fleet-package-registry
     - hostname: Fleet-package-reg-{{ GLOBALS.hostname }}
     - detach: True

salt/elasticagent/enabled.sls

@@ -7,15 +7,15 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-
+{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 include:
   - elasticagent.config
   - elasticagent.sostatus
 
 so-elastic-agent:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
     - name: so-elastic-agent
     - hostname: {{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/enabled.sls

@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
@@ -71,7 +72,7 @@ elasticagent_syncartifacts:
 {%   if SERVICETOKEN != '' %}
 so-elastic-fleet:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
     - name: so-elastic-fleet
     - hostname: FleetServer-{{ GLOBALS.hostname }}
     - detach: True

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -7,6 +7,7 @@
 #so-elastic-agent-gen-installers $FleetHostURLs $EnrollmentToken 
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS -%}
 
 . /usr/sbin/so-common
 . /usr/sbin/so-elastic-fleet-common
@@ -37,7 +38,7 @@ rm -rf /nsm/elastic-agent-workspace
 mkdir -p /nsm/elastic-agent-workspace
 
 printf "\n### Extracting outer tarball and then each individual tarball/zip\n"
-tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-agent-workspace/
+tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}.tar.gz -C /nsm/elastic-agent-workspace/
 unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
 for archive in /nsm/elastic-agent-workspace/*.tar.gz
 do

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -5,6 +5,7 @@
 # this file except in compliance with the Elastic License 2.0.
 
 . /usr/sbin/so-common
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS -%}
 
 # Only run on Managers
 if ! is_manager_node; then
@@ -27,14 +28,14 @@ OUTDATED_LIST=$(jq -r '.items | map(.id) | (tojson)'  <<<  "$RAW_JSON")
 
 if [ "$OUTDATED_LIST" != '[]' ]; then
    AGENTNUMBERS=$(jq -r '.total' <<< "$RAW_JSON")
-   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic $ELASTIC_AGENT_TARBALL_VERSION...\n\n"
+   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"
 
    # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION $ELASTIC_AGENT_TARBALL_VERSION --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
 
    # Update Node Agents
    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 else
     printf "No Agents need updates... Exiting\n\n"
     exit 0
-fi
+fi

salt/elasticsearch/defaults.yaml

@@ -1,5 +1,6 @@
 elasticsearch:
   enabled: false
+  version: 8.14.3
   index_clean: true
   config:
     action:

salt/elasticsearch/enabled.sls

@@ -19,7 +19,7 @@ include:
 
 so-elasticsearch:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.elasticsearch.version }}
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch

salt/elasticsearch/soc_elasticsearch.yaml

@@ -2,6 +2,11 @@ elasticsearch:
   enabled:
     description: You can enable or disable Elasticsearch.
     helpLink: elasticsearch.html
+  version:
+    description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
+    readonly: True
+    global: True
+    advanced: True
   esheap:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html

salt/kibana/enabled.sls

@@ -7,6 +7,7 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 include:
   - kibana.config
@@ -15,7 +16,7 @@ include:
 # Start the kibana docker
 so-kibana:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kibana:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
     - hostname: kibana
     - user: kibana
     - networks:

salt/kibana/files/config_saved_objects.ndjson.jinja

@@ -0,0 +1,3 @@
+{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS -%}
+
+{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","id": "{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/kibana/so_config_load.sls

@@ -9,7 +9,8 @@ include:
 config_saved_objects:
   file.managed:
     - name: /opt/so/conf/kibana/config_saved_objects.ndjson.template
-    - source: salt://kibana/files/config_saved_objects.ndjson
+    - source: salt://kibana/files/config_saved_objects.ndjson.jinja
+    - template: jinja
     - user: 932
     - group: 939
 

salt/kibana/tools/sbin_jinja/so-kibana-config-load

@@ -5,6 +5,8 @@
 # Elastic License 2.0.
 
 {%- set ENDGAMEHOST = salt['pillar.get']('global:endgamehost', 'ENDGAMEHOST') %}
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
+
 . /usr/sbin/so-common
 
 check_file() {
@@ -63,7 +65,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
 

salt/logstash/enabled.sls

@@ -10,6 +10,7 @@
 {%   from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 {%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
 {%   set lsheap = LOGSTASH_MERGED.settings.lsheap %}
+{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 include:
 {%   if GLOBALS.role not in ['so-receiver','so-fleet'] %}
@@ -26,7 +27,7 @@ include:
 
 so-logstash:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-logstash:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
     - hostname: so-logstash
     - name: so-logstash
     - networks:

salt/manager/tools/sbin/soup

@@ -842,6 +842,7 @@ determine_elastic_agent_upgrade() {
 }
 
 update_elastic_agent_airgap() {
+  get_elastic_agent_vars
   rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/
   tar -xf "$ELASTIC_AGENT_FILE" -C "$ELASTIC_AGENT_EXPANSION_DIR"
 }