Merge pull request #4003 from Security-Onion-Solutions/dev

2.3.50

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.40
+## Security Onion 2.3.50
 
-Security Onion 2.3.40 is here!
+Security Onion 2.3.50 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,16 +1,17 @@
-### 2.3.40 ISO image built on 2021/03/22
+### 2.3.50 ISO image built on 2021/04/27
+
 
 ### Download and Verify
 
-2.3.40 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
+2.3.50 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso
 
-MD5: FB72C0675F262A714B287BB33CE82504  
-SHA1: E8F5A9AA23990DF794611F9A178D88414F5DA81C  
-SHA256: DB125D6E770F75C3FD35ABE3F8A8B21454B7A7618C2B446D11B6AC8574601070 
+MD5: C39CEA68B5A8AFC5CFFB2481797C0374  
+SHA1: 00AD9F29ABE3AB495136989E62EBB8FA00DA82C6  
+SHA256: D77AE370D7863837A989F6735413D1DD46B866D8D135A4C363B0633E3990387E 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.50.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.50.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.40.iso.sig securityonion-2.3.40.iso
+gpg --verify securityonion-2.3.50.iso.sig securityonion-2.3.50.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 22 Mar 2021 09:35:50 AM EDT using RSA key ID FE507013
+gpg: Signature made Tue 27 Apr 2021 02:17:25 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <[email protected]>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.40
+2.3.50

salt/ca/init.sls

@@ -43,8 +43,9 @@ pki_private_key:
     - require:
       - file: /etc/pki
     - timeout: 30
-    - retry: 5
-    - interval: 30
+    - retry:
+        attempts: 5
+        interval: 30
 
 x509_pem_entries:
   module.run:

salt/common/files/99-reserved-ports.conf

@@ -1 +1 @@
-net.ipv4.ip_local_reserved_ports=55000,57314
+net.ipv4.ip_local_reserved_ports=55000,57314,47760-47860

salt/common/files/soversion

@@ -0,0 +1,2 @@
+{%- set VERSION = salt['pillar.get']('global:soversion') -%}
+{{ VERSION }}

salt/common/files/vimrc

@@ -0,0 +1,6 @@
+" Activates filetype detection
+filetype plugin indent on
+
+" Sets .sls files to use YAML syntax highlighting
+autocmd BufNewFile,BufRead *.sls set syntax=yaml
+set number

salt/common/init.sls

@@ -49,6 +49,11 @@ sosaltstackperms:
     - gid: 939
     - dir_mode: 770
 
+so_log_perms:
+  file.directory:
+    - name: /opt/so/log
+    - dir_mode: 755
+
 # Create a state directory
 statedir:
   file.directory:
@@ -64,20 +69,12 @@ salttmp:
     - group: 939
     - makedirs: True
 
-# Install epel
-{% if grains['os'] == 'CentOS' %}
-repair_yumdb:
-  cmd.run:
-    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
-    - onlyif:
-      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
-
-epel:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - epel-release
-{% endif %}
+# VIM config
+vimconfig:
+  file.managed:
+    - name: /root/.vimrc
+    - source: salt://common/files/vimrc
+    - replace: False
 
 # Install common packages
 {% if grains['os'] != 'CentOS' %}     
@@ -105,6 +102,8 @@ commonpkgs:
       - python3-mysqldb
       - python3-packaging
       - git
+      - vim
+
 heldpackages:
   pkg.installed:
     - pkgs:
@@ -143,6 +142,7 @@ commonpkgs:
       - lvm2
       - openssl
       - git
+      - vim-enhanced
 
 heldpackages:
   pkg.installed:
@@ -235,6 +235,30 @@ commonlogrotateconf:
     - month: '*'
     - dayweek: '*'
 
+# Create the status directory
+sostatusdir:
+  file.directory:
+    - name: /opt/so/log/sostatus
+    - user: 0
+    - group: 0
+    - makedirs: True
+
+sostatus_log:
+  file.managed:
+    - name: /opt/so/log/sostatus/status.log
+    - mode: 644
+    
+# Install sostatus check cron
+'/usr/sbin/so-status -q; echo $? > /opt/so/log/sostatus/status.log 2>&1':
+  cron.present:
+    - user: root
+    - minute: '*/1'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+
 {% if role in ['eval', 'manager', 'managersearch', 'standalone'] %}
 # Lock permissions on the backup directory
 backupdir:
@@ -254,6 +278,14 @@ backupdir:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
+{% else %}
+soversionfile:
+  file.managed:
+    - name: /etc/soversion
+    - source: salt://common/files/soversion
+    - mode: 644
+    - template: jinja
+    
 {% endif %}
 
 # Manager daemon.json
@@ -271,9 +303,10 @@ docker:
       - file: docker_daemon
 
 # Reserve OS ports for Docker proxy in case boot settings are not already applied/present
+# 55000 = Wazuh, 57314 = Strelka, 47760-47860 = Zeek
 dockerapplyports:
     cmd.run:
-      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314"; fi
+      - name: if [ ! -s /etc/sysctl.d/99-reserved-ports.conf ]; then sysctl -w net.ipv4.ip_local_reserved_ports="55000,57314,47760-47860"; fi
 
 # Reserve OS ports for Docker proxy
 dockerreserveports:

salt/common/tools/sbin/so-common

@@ -122,6 +122,10 @@ check_elastic_license() {
 	fi  
 }
 
+disable_fastestmirror() {
+	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -162,6 +166,23 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+gpg_rpm_import() {
+	if [[ "$OS" == "centos" ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/centos/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys"
+	    fi
+
+	    RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub')
+
+	    for RPMKEY in "${RPMKEYS[@]}"; do
+    	        rpm --import $RPMKEYSLOC/$RPMKEY
+	        echo "Imported $RPMKEY"
+	    done
+	fi
+}
+
 header() {
 	printf '%s\n' "" "$banner" " $*" "$banner" 
 }
@@ -419,6 +440,20 @@ valid_proxy() {
 	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
 }
 
+valid_ntp_list() {
+	local string=$1
+	local ntp_arr
+	IFS="," read -r -a ntp_arr <<< "$string"
+
+	for ntp in "${ntp_arr[@]}"; do
+		if ! valid_ip4 "$ntp" && ! valid_hostname "$ntp" && ! valid_fqdn "$ntp"; then
+			return 1
+		fi
+	done
+
+	return 0
+}
+
 valid_string() {
 	local str=$1
 	local min_length=${2:-1}

salt/common/tools/sbin/so-docker-prune

@@ -60,15 +60,19 @@ def main(quiet):
   no_prunable = True
   for t_list in grouped_tag_lists:
     try:
-      # Keep the 2 most current images
+      # Group tags by version, in case multiple images exist with the same version string
       t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
-      if len(t_list) <= 2:
+      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
+
+      # Keep the 2 most current version groups
+      if len(grouped_t_list) <= 2:
         continue
       else:
         no_prunable = False
-        for tag in t_list[2:]:
-          if not quiet: print(f'Removing image {tag}')
-          client.images.remove(tag)
+        for group in grouped_t_list[2:]:
+          for tag in group:
+            if not quiet: print(f'Removing image {tag}')
+            client.images.remove(tag)
     except InvalidVersion as e:
       print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
       exit(1)

salt/common/tools/sbin/so-elasticsearch-indices-list

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+
+. /usr/sbin/so-common
+
+curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty