Merge pull request #3572 from Security-Onion-Solutions/dev

2.3.40

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.30
+## Security Onion 2.3.40
 
-Security Onion 2.3.30 is here!
+Security Onion 2.3.40 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,16 +1,16 @@
-### 2.3.30 ISO image built on 2021/03/01
+### 2.3.40 ISO image built on 2021/03/22
 
 ### Download and Verify
 
-2.3.30 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.30.iso
+2.3.40 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
 
-MD5: 65202BA0F7661A5E27087F097B8E571E  
-SHA1: 14E842E39EDBB55A104263281CF25BF88A2E9D67  
-SHA256: 210B37B9E3DFC827AFE2940E2C87B175ADA968EDD04298A5926F63D9269847B7 
+MD5: FB72C0675F262A714B287BB33CE82504  
+SHA1: E8F5A9AA23990DF794611F9A178D88414F5DA81C  
+SHA256: DB125D6E770F75C3FD35ABE3F8A8B21454B7A7618C2B446D11B6AC8574601070 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.30.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +24,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.30.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.30.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.30.iso.sig securityonion-2.3.30.iso
+gpg --verify securityonion-2.3.40.iso.sig securityonion-2.3.40.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 01 Mar 2021 02:15:28 PM EST using RSA key ID FE507013
+gpg: Signature made Mon 22 Mar 2021 09:35:50 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <[email protected]>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.30
+2.3.40

pillar/logstash/search.sls

@@ -7,7 +7,7 @@ logstash:
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
-        - so/9100_output_osquery.conf.jinja
+        - so/9100_output_osquery.conf.jinja  
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja

salt/common/files/99-reserved-ports.conf

@@ -1 +1 @@
-net.ipv4.ip_local_reserved_ports="55000,57314"
+net.ipv4.ip_local_reserved_ports=55000,57314

salt/common/init.sls

@@ -90,7 +90,6 @@ commonpkgs:
       - ntpdate
       - jq
       - python3-docker
-      - docker-ce
       - curl
       - ca-certificates
       - software-properties-common
@@ -104,12 +103,15 @@ commonpkgs:
       - python3-dateutil
       - python3-m2crypto
       - python3-mysqldb
+      - python3-packaging
       - git
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-2
-      - docker-ce: 5:19.03.14~3-0~ubuntu-bionic
+      - containerd.io: 1.4.4-1
+      - docker-ce: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-bionic
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-bionic
     - hold: True
     - update_holds: True
 
@@ -135,6 +137,7 @@ commonpkgs:
       - python36-dateutil
       - python36-m2crypto
       - python36-mysql
+      - python36-packaging
       - yum-utils
       - device-mapper-persistent-data
       - lvm2
@@ -144,8 +147,10 @@ commonpkgs:
 heldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.2.13-3.2.el7
-      - docker-ce: 3:19.03.14-3.el7
+      - containerd.io: 1.4.4-3.1.el7
+      - docker-ce: 3:20.10.5-3.el7
+      - docker-ce-cli: 1:20.10.5-3.el7
+      - docker-ce-rootless-extras: 20.10.5-3.el7
     - hold: True
     - update_holds: True
 {% endif %}

salt/common/tools/sbin/so-common

@@ -86,6 +86,19 @@ add_interface_bond0() {
 	fi
 }
 
+check_airgap() {
+  # See if this is an airgap install
+  AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
+  if [[ "$AIRGAP" == "True" ]]; then
+      is_airgap=0
+      UPDATE_DIR=/tmp/soagupdate/SecurityOnion
+      AGDOCKER=/tmp/soagupdate/docker
+      AGREPO=/tmp/soagupdate/Packages
+  else 
+      is_airgap=1
+  fi
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -97,6 +110,46 @@ check_password() {
 	return $?
 }
 
+check_elastic_license() {
+
+	[ -n "$TESTING" ] && return
+
+	# See if the user has already accepted the license
+	if [ ! -f /opt/so/state/yeselastic.txt ]; then
+	  elastic_license
+	else
+	  echo "Elastic License has already been accepted"
+	fi  
+}
+
+elastic_license() {
+
+read -r -d '' message <<- EOM
+\n
+Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
+https://securityonion.net/elastic-license
+
+Please review the Elastic License:
+https://www.elastic.co/licensing/elastic-license
+
+Do you agree to the terms of the Elastic License?
+
+If so, type AGREE to accept the Elastic License and continue.  Otherwise, press Enter to exit this program without making any changes.
+EOM
+
+AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
+"$message" 20 75 3>&1 1>&2 2>&3)
+
+if [ "${AGREED^^}" = 'AGREE' ]; then
+  mkdir -p /opt/so/state
+  touch /opt/so/state/yeselastic.txt 
+else
+  echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
+  exit 1
+fi
+
+}
+
 fail() {
 	msg=$1
 	echo "ERROR: $msg"
@@ -250,6 +303,12 @@ set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
+set_palette() {
+	if [ "$OS" == ubuntu ]; then
+	    update-alternatives --set newt-palette /etc/newt/palette.original
+    fi
+}
+
 set_version() {
 	CURRENTVERSION=0.0.0
 	if [ -f /etc/soversion ]; then
@@ -340,6 +399,26 @@ valid_int() {
 
 # {% raw %}
 
+valid_proxy() {
+	local proxy=$1
+	local url_prefixes=( 'http://' 'https://' )
+
+	local has_prefix=false
+	for prefix in "${url_prefixes[@]}"; do
+		echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break
+	done
+
+	local url_arr
+	mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")"
+
+	local valid_url=true
+	if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}" && ! valid_hostname "${url_arr[0]}"; then
+		valid_url=false
+	fi
+
+	[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
+}
+
 valid_string() {
 	local str=$1
 	local min_length=${2:-1}

salt/common/tools/sbin/so-cortex-user-add

@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
 CORTEX_USER=$USER

salt/common/tools/sbin/so-cortex-user-enable

@@ -30,7 +30,7 @@ fi
 
 USER=$1
 
-CORTEX_KEY=$(lookup_pillar cortexkey)
+CORTEX_KEY=$(lookup_pillar cortexorguserkey)
 CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
 CORTEX_USER=$USER
 

salt/common/tools/sbin/so-docker-prune

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys, argparse, re, docker
+from packaging.version import Version, InvalidVersion
+from itertools import groupby, chain
+
+
+def get_image_name(string) -> str:
+  return ':'.join(string.split(':')[:-1])
+
+
+def get_so_image_basename(string) -> str:
+  return get_image_name(string).split('/so-')[-1]
+
+
+def get_image_version(string) -> str:
+  ver = string.split(':')[-1]
+  if ver == 'latest':
+    # Version doesn't like "latest", so use a high semver
+    return '999999.9.9'
+  else:
+    try:
+      Version(ver)
+    except InvalidVersion:
+      # Strip the last substring following a hyphen for automated branches
+      ver = '-'.join(ver.split('-')[:-1])  
+    return ver
+
+
+def main(quiet):
+  client = docker.from_env()
+
+  image_list = client.images.list(filters={ 'dangling': False })
+
+  # Map list of image objects to flattened list of tags (format: "name:version")
+  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
+
+  # Filter to only SO images (base name begins with "so-")
+  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
+
+  # Group tags into lists by base name (sort by same projection first)
+  tag_list.sort(key=lambda x: get_so_image_basename(x))
+  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
+
+  no_prunable = True
+  for t_list in grouped_tag_lists:
+    try:
+      # Keep the 2 most current images
+      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+      if len(t_list) <= 2:
+        continue
+      else:
+        no_prunable = False
+        for tag in t_list[2:]:
+          if not quiet: print(f'Removing image {tag}')
+          client.images.remove(tag)
+    except InvalidVersion as e:
+      print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
+      exit(1)
+
+  if no_prunable and not quiet:
+    print('No Security Onion images to prune')
+
+
+if __name__ == "__main__":
+  main_parser = argparse.ArgumentParser(add_help=False)
+  main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
+  args = main_parser.parse_args(sys.argv[1:])
+
+  main(args.quiet)

salt/common/tools/sbin/so-elastic-clear

@@ -50,11 +50,7 @@ done
 if [ $SKIP -ne 1 ]; then
         # List indices
         echo 
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        {% else %}
-        curl -L {{ NODEIP }}:9200/_cat/indices?v
-        {% endif %}
         echo
         # Inform user we are about to delete all data
         echo
@@ -93,18 +89,10 @@ fi
 # Delete data
 echo "Deleting data..."
 
-{% if grains['role'] in ['so-node','so-heavynode'] %}
 INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% else %}
-INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-{% endif %}
 for INDX in ${INDXS}
 do
-    {% if grains['role'] in ['so-node','so-heavynode'] %}
     curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% else %}
-    curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-    {% endif %}
 done
 
 #Start Logstash/Filebeat

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
 
 echo "Removing read only attributes for indices..."
 echo
-for p in $ESPORT $THEHIVEESPORT; do
-   curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
-done
+curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
+curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-elasticsearch-pipeline-stats

@@ -19,15 +19,7 @@
 . /usr/sbin/so-common
 
 if [ "$1" == "" ]; then
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
-        {% endif %}
 else
-        {% if grains['role'] in ['so-node','so-heavynode'] %}
         curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% else %}
-        curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
-        {% endif %}
 fi